diff --git a/Readme.md b/Readme.md index 8f91114..96220b6 100644 --- a/Readme.md +++ b/Readme.md @@ -85,18 +85,7 @@ For more information checkout [Setting Alarm](http://naver.github.io/pinpoint/al The Pinpoint-Flink is necessary to use ['Application Inspector'](http://naver.github.io/pinpoint/applicationinspector.html) feature. -After all containers are started and ready to go. There is one more thing to do to use this features in Pinpoint. -This feature is not mandatory, but since enabling this is a simple task, let's take care of it. - -Register a `job` on to pinpoint-flink server. -You can build the `job` from the [open-source of Pinpoint](https://github.com/naver/pinpoint), additional guide is [here](https://github.com/naver/pinpoint/blob/master/doc/application-inspector.md#application-inspector). - -If anyone have solution to put the job file into flink image without doing manually, please let us know. - -Pinpoint-Flink server is running on [port 8081](http://localhost:8081/#/submit). From `submit new job` menu -Submit the jar file with *com.navercorp.pinpoint.flink.StatStreamingVer2Job* in entry class as below image. - -![Pinpoint](https://github.com/naver/pinpoint-docker/blob/master/docs/Pinpoint-Flink%20upload.png) +After all containers are started and ready to go. Pinpoint-Flink server is running on [port 8081](http://localhost:8081/). ### Alarm configuration (optional) diff --git a/docker-compose.yml b/docker-compose.yml index e2bf3f5..8fdd39d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -249,23 +249,30 @@ services: ##flink jobmanager: + build: + context: pinpoint-flink + dockerfile: Dockerfile container_name: "${PINPOINT_FLINK_NAME}-jobmanager" - image: flink:1.3.1 + image: ${PINPOINT_FLINK_NAME}-flink:dev expose: - "6123" ports: - "${FLINK_WEB_PORT:-8081}:8081" - command: jobmanager + command: standalone-job -p 1 pinpoint-flink-job.jar -spring.profiles.active release environment: - JOB_MANAGER_RPC_ADDRESS=jobmanager + - PINPOINT_ZOOKEEPER_ADDRESS=${PINPOINT_ZOOKEEPER_ADDRESS} networks: - pinpoint depends_on: - zoo1 taskmanager: + build: + context: pinpoint-flink + dockerfile: Dockerfile container_name: "${PINPOINT_FLINK_NAME}-taskmanager" - image: flink:1.3.1 + image: ${PINPOINT_FLINK_NAME}-flink:dev expose: - "6121" - "6122" diff --git a/docs/Pinpoint-Flink upload.png b/docs/Pinpoint-Flink upload.png deleted file mode 100644 index 8bb7a58..0000000 Binary files a/docs/Pinpoint-Flink upload.png and /dev/null differ diff --git a/pinpoint-flink/.env b/pinpoint-flink/.env index ae123a0..023e73a 100644 --- a/pinpoint-flink/.env +++ b/pinpoint-flink/.env @@ -2,6 +2,7 @@ PINPOINT_VERSION=2.2.1 ### Pinpoint-flink +PINPOINT_ZOOKEEPER_ADDRESS=zookeeper PINPOINT_FLINK_NAME=pinpoint-flink FLINK_WEB_PORT=8081 diff --git a/pinpoint-flink/Dockerfile b/pinpoint-flink/Dockerfile new file mode 100644 index 0000000..ecb0f7c --- /dev/null +++ b/pinpoint-flink/Dockerfile @@ -0,0 +1,11 @@ +FROM flink:1.3 +ARG PINPOINT_VERSION=2.2.0 +# hadolint ignore=DL3008 +RUN apt-get update \ + && apt-get install --no-install-recommends -y zip \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* +ADD https://github.com/pinpoint-apm/pinpoint/releases/download/v${PINPOINT_VERSION}/pinpoint-flink-job-${PINPOINT_VERSION}.jar /build/ +COPY build/ /build/ +COPY docker-bin/ /docker-bin/ +ENTRYPOINT ["/docker-bin/docker-entrypoint.sh"] \ No newline at end of file diff --git a/pinpoint-flink/build/hbase.properties b/pinpoint-flink/build/profiles/release/hbase.properties similarity index 55% rename from pinpoint-flink/build/hbase.properties rename to pinpoint-flink/build/profiles/release/hbase.properties index 19a80ff..36c3188 100644 --- a/pinpoint-flink/build/hbase.properties +++ b/pinpoint-flink/build/profiles/release/hbase.properties @@ -1,21 +1,4 @@ -# -# Copyright 2017 NAVER Corp. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# local -hbase.client.host=pinpoint-hbase +hbase.client.host=#PINPOINT_ZOOKEEPER_ADDRESS# hbase.client.port=2181 # hbase default:/hbase diff --git a/pinpoint-flink/build/profiles/release/pinpoint-collector.properties b/pinpoint-flink/build/profiles/release/pinpoint-collector.properties new file mode 100644 index 0000000..7338fed --- /dev/null +++ b/pinpoint-flink/build/profiles/release/pinpoint-collector.properties @@ -0,0 +1 @@ +pinpoint.zookeeper.address=#PINPOINT_ZOOKEEPER_ADDRESS# \ No newline at end of file diff --git a/pinpoint-flink/build/pinpoint-flink.properties b/pinpoint-flink/build/profiles/release/pinpoint-flink.properties similarity index 75% rename from pinpoint-flink/build/pinpoint-flink.properties rename to pinpoint-flink/build/profiles/release/pinpoint-flink.properties index b3f1899..78f4a41 100644 --- a/pinpoint-flink/build/pinpoint-flink.properties +++ b/pinpoint-flink/build/profiles/release/pinpoint-flink.properties @@ -2,7 +2,7 @@ collector.l4.ip= # base data receiver config --------------------------------------------------------------------- -collector.receiver.base.ip=pinpoint-flink-taskmanager +collector.receiver.base.ip=0.0.0.0 collector.receiver.base.port=19994 # number of tcp worker threads @@ -23,26 +23,24 @@ collector.agentEventWorker.threadSize=4 # capacity of agent event worker queue collector.agentEventWorker.queueSize=1024 -statistics.flushPeriod=1000 - # ------------------------------------------------------------------------------------------------- # The cluster related options are used to establish connections between the agent, collector, and web in order to send/receive data between them in real time. # You may enable additional features using this option (Ex : RealTime Active Thread Chart). # ------------------------------------------------------------------------------------------------- # Usage : Set the following options for collector/web components that reside in the same cluster in order to enable this feature. -# 1. cluster.enable (pinpoint-web.properties, pinpoint-flink.properties) - "true" to enable -# 2. cluster.zookeeper.address (pinpoint-web.properties, pinpoint-flink.properties) - address of the ZooKeeper instance that will be used to manage the cluster -# 3. cluster.web.tcp.port (pinpoint-web.properties) - any available port number (used to establish connection between web and collector) +# 1. cluster.enable (pinpoint-web-root.properties, pinpoint-flink.properties) - "true" to enable +# 2. cluster.zookeeper.address (pinpoint-web-root.properties, pinpoint-flink.properties) - address of the ZooKeeper instance that will be used to manage the cluster +# 3. cluster.web.tcp.port (pinpoint-web-root.properties) - any available port number (used to establish connection between web and collector) # ------------------------------------------------------------------------------------------------- # Please be aware of the following: #1. If the network between web, collector, and the agents are not stable, it is advisable not to use this feature. #2. We recommend using the cluster.web.tcp.port option. However, in cases where the collector is unable to establish connection to the web, you may reverse this and make the web establish connection to the collector. -# In this case, you must set cluster.connect.address (pinpoint-web.properties); and cluster.listen.ip, cluster.listen.port (pinpoint-flink.properties) accordingly. -cluster.enable=true -cluster.zookeeper.address=zoo1 +# In this case, you must set cluster.connect.address (pinpoint-web-root.properties); and cluster.listen.ip, cluster.listen.port (pinpoint-flink.properties) accordingly. +cluster.enable=false +cluster.zookeeper.address=#PINPOINT_ZOOKEEPER_ADDRESS# cluster.zookeeper.sessiontimeout=30000 cluster.listen.ip= -cluster.listen.port= +cluster.listen.port=-1 #collector.admin.password= #collector.admin.api.rest.active= @@ -52,11 +50,10 @@ collector.spanEvent.sequence.limit=10000 # flink cluster flink.cluster.enable=true -flink.cluster.zookeeper.address=zoo1 +flink.cluster.zookeeper.address=#PINPOINT_ZOOKEEPER_ADDRESS# flink.cluster.zookeeper.sessiontimeout=3000 flink.cluster.zookeeper.retry.interval=5000 flink.cluster.tcp.port=19994 # flink env init -flink.StreamExecutionEnvironment= -flink.sourceFunction.Parallel=1 +flink.StreamExecutionEnvironment=server \ No newline at end of file diff --git a/pinpoint-flink/build/profiles/release/pinpoint-web.properties b/pinpoint-flink/build/profiles/release/pinpoint-web.properties new file mode 100644 index 0000000..7338fed --- /dev/null +++ b/pinpoint-flink/build/profiles/release/pinpoint-web.properties @@ -0,0 +1 @@ +pinpoint.zookeeper.address=#PINPOINT_ZOOKEEPER_ADDRESS# \ No newline at end of file diff --git a/pinpoint-flink/docker-bin/docker-entrypoint.sh b/pinpoint-flink/docker-bin/docker-entrypoint.sh new file mode 100755 index 0000000..67676ee --- /dev/null +++ b/pinpoint-flink/docker-bin/docker-entrypoint.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# In flink version < 1.11 we can't use "job cluster" because it's not yet available +if [ "$1" = "standalone-job" ]; then + cd /build + sed -i -e "s/#PINPOINT_ZOOKEEPER_ADDRESS#/${PINPOINT_ZOOKEEPER_ADDRESS}/g" profiles/release/* + zip -r pinpoint-flink-job-*.jar profiles/ + cd - + + mv /build/pinpoint-flink-job-*.jar pinpoint-flink-job.jar + + shift + + /docker-entrypoint.sh jobmanager & + sleep 5 + exec flink run $@ +else + exec /docker-entrypoint.sh $@ +fi \ No newline at end of file diff --git a/pinpoint-flink/docker-compose.yml b/pinpoint-flink/docker-compose.yml index f1c7c14..f5350f6 100644 --- a/pinpoint-flink/docker-compose.yml +++ b/pinpoint-flink/docker-compose.yml @@ -7,21 +7,24 @@ version: "3.6" services: ##flink jobmanager: + build: Dockerfile container_name: "${PINPOINT_FLINK_NAME}-jobmanager" - image: flink:1.3.1 + image: ${PINPOINT_FLINK_NAME}-flink:dev expose: - "6123" ports: - "${FLINK_WEB_PORT:-8081}:8081" - command: jobmanager + command: standalone-job -p 1 pinpoint-flink-job.jar -spring.profiles.active release environment: - JOB_MANAGER_RPC_ADDRESS=jobmanager + - PINPOINT_ZOOKEEPER_ADDRESS=${PINPOINT_ZOOKEEPER_ADDRESS} networks: - pinpoint taskmanager: + build: Dockerfile container_name: "${PINPOINT_FLINK_NAME}-taskmanager" - image: flink:1.3.1 + image: ${PINPOINT_FLINK_NAME}-flink:dev expose: - "6121" - "6122"