Load job when jobmanager startup

This commit is contained in:
Charles DELFLY 2021-01-13 14:58:02 +07:00 committed by Roy Kim
parent aeedca1a15
commit 77b55c8e53
11 changed files with 61 additions and 49 deletions

View File

@ -85,18 +85,7 @@ For more information checkout [Setting Alarm](http://naver.github.io/pinpoint/al
The Pinpoint-Flink is necessary to use ['Application Inspector'](http://naver.github.io/pinpoint/applicationinspector.html) feature.
After all containers are started and ready to go. There is one more thing to do to use this features in Pinpoint.
This feature is not mandatory, but since enabling this is a simple task, let's take care of it.
Register a `job` on to pinpoint-flink server.
You can build the `job` from the [open-source of Pinpoint](https://github.com/naver/pinpoint), additional guide is [here](https://github.com/naver/pinpoint/blob/master/doc/application-inspector.md#application-inspector).
If anyone have solution to put the job file into flink image without doing manually, please let us know.
Pinpoint-Flink server is running on [port 8081](http://localhost:8081/#/submit). From `submit new job` menu
Submit the jar file with *com.navercorp.pinpoint.flink.StatStreamingVer2Job* in entry class as below image.
![Pinpoint](https://github.com/naver/pinpoint-docker/blob/master/docs/Pinpoint-Flink%20upload.png)
After all containers are started and ready to go. Pinpoint-Flink server is running on [port 8081](http://localhost:8081/).
### Alarm configuration (optional)

View File

@ -249,23 +249,30 @@ services:
##flink
jobmanager:
build:
context: pinpoint-flink
dockerfile: Dockerfile
container_name: "${PINPOINT_FLINK_NAME}-jobmanager"
image: flink:1.3.1
image: ${PINPOINT_FLINK_NAME}-flink:dev
expose:
- "6123"
ports:
- "${FLINK_WEB_PORT:-8081}:8081"
command: jobmanager
command: standalone-job -p 1 pinpoint-flink-job.jar -spring.profiles.active release
environment:
- JOB_MANAGER_RPC_ADDRESS=jobmanager
- PINPOINT_ZOOKEEPER_ADDRESS=${PINPOINT_ZOOKEEPER_ADDRESS}
networks:
- pinpoint
depends_on:
- zoo1
taskmanager:
build:
context: pinpoint-flink
dockerfile: Dockerfile
container_name: "${PINPOINT_FLINK_NAME}-taskmanager"
image: flink:1.3.1
image: ${PINPOINT_FLINK_NAME}-flink:dev
expose:
- "6121"
- "6122"

Binary file not shown.

Before

Width:  |  Height:  |  Size: 346 KiB

View File

@ -2,6 +2,7 @@ PINPOINT_VERSION=2.2.1
### Pinpoint-flink
PINPOINT_ZOOKEEPER_ADDRESS=zookeeper
PINPOINT_FLINK_NAME=pinpoint-flink
FLINK_WEB_PORT=8081

11
pinpoint-flink/Dockerfile Normal file
View File

@ -0,0 +1,11 @@
FROM flink:1.3
ARG PINPOINT_VERSION=2.2.0
# hadolint ignore=DL3008
RUN apt-get update \
&& apt-get install --no-install-recommends -y zip \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
ADD https://github.com/pinpoint-apm/pinpoint/releases/download/v${PINPOINT_VERSION}/pinpoint-flink-job-${PINPOINT_VERSION}.jar /build/
COPY build/ /build/
COPY docker-bin/ /docker-bin/
ENTRYPOINT ["/docker-bin/docker-entrypoint.sh"]

View File

@ -1,21 +1,4 @@
#
# Copyright 2017 NAVER Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# local
hbase.client.host=pinpoint-hbase
hbase.client.host=#PINPOINT_ZOOKEEPER_ADDRESS#
hbase.client.port=2181
# hbase default:/hbase

View File

@ -0,0 +1 @@
pinpoint.zookeeper.address=#PINPOINT_ZOOKEEPER_ADDRESS#

View File

@ -2,7 +2,7 @@
collector.l4.ip=
# base data receiver config ---------------------------------------------------------------------
collector.receiver.base.ip=pinpoint-flink-taskmanager
collector.receiver.base.ip=0.0.0.0
collector.receiver.base.port=19994
# number of tcp worker threads
@ -23,26 +23,24 @@ collector.agentEventWorker.threadSize=4
# capacity of agent event worker queue
collector.agentEventWorker.queueSize=1024
statistics.flushPeriod=1000
# -------------------------------------------------------------------------------------------------
# The cluster related options are used to establish connections between the agent, collector, and web in order to send/receive data between them in real time.
# You may enable additional features using this option (Ex : RealTime Active Thread Chart).
# -------------------------------------------------------------------------------------------------
# Usage : Set the following options for collector/web components that reside in the same cluster in order to enable this feature.
# 1. cluster.enable (pinpoint-web.properties, pinpoint-flink.properties) - "true" to enable
# 2. cluster.zookeeper.address (pinpoint-web.properties, pinpoint-flink.properties) - address of the ZooKeeper instance that will be used to manage the cluster
# 3. cluster.web.tcp.port (pinpoint-web.properties) - any available port number (used to establish connection between web and collector)
# 1. cluster.enable (pinpoint-web-root.properties, pinpoint-flink.properties) - "true" to enable
# 2. cluster.zookeeper.address (pinpoint-web-root.properties, pinpoint-flink.properties) - address of the ZooKeeper instance that will be used to manage the cluster
# 3. cluster.web.tcp.port (pinpoint-web-root.properties) - any available port number (used to establish connection between web and collector)
# -------------------------------------------------------------------------------------------------
# Please be aware of the following:
#1. If the network between web, collector, and the agents are not stable, it is advisable not to use this feature.
#2. We recommend using the cluster.web.tcp.port option. However, in cases where the collector is unable to establish connection to the web, you may reverse this and make the web establish connection to the collector.
# In this case, you must set cluster.connect.address (pinpoint-web.properties); and cluster.listen.ip, cluster.listen.port (pinpoint-flink.properties) accordingly.
cluster.enable=true
cluster.zookeeper.address=zoo1
# In this case, you must set cluster.connect.address (pinpoint-web-root.properties); and cluster.listen.ip, cluster.listen.port (pinpoint-flink.properties) accordingly.
cluster.enable=false
cluster.zookeeper.address=#PINPOINT_ZOOKEEPER_ADDRESS#
cluster.zookeeper.sessiontimeout=30000
cluster.listen.ip=
cluster.listen.port=
cluster.listen.port=-1
#collector.admin.password=
#collector.admin.api.rest.active=
@ -52,11 +50,10 @@ collector.spanEvent.sequence.limit=10000
# flink cluster
flink.cluster.enable=true
flink.cluster.zookeeper.address=zoo1
flink.cluster.zookeeper.address=#PINPOINT_ZOOKEEPER_ADDRESS#
flink.cluster.zookeeper.sessiontimeout=3000
flink.cluster.zookeeper.retry.interval=5000
flink.cluster.tcp.port=19994
# flink env init
flink.StreamExecutionEnvironment=
flink.sourceFunction.Parallel=1
flink.StreamExecutionEnvironment=server

View File

@ -0,0 +1 @@
pinpoint.zookeeper.address=#PINPOINT_ZOOKEEPER_ADDRESS#

View File

@ -0,0 +1,19 @@
#!/bin/bash
# In flink version < 1.11 we can't use "job cluster" because it's not yet available
if [ "$1" = "standalone-job" ]; then
cd /build
sed -i -e "s/#PINPOINT_ZOOKEEPER_ADDRESS#/${PINPOINT_ZOOKEEPER_ADDRESS}/g" profiles/release/*
zip -r pinpoint-flink-job-*.jar profiles/
cd -
mv /build/pinpoint-flink-job-*.jar pinpoint-flink-job.jar
shift
/docker-entrypoint.sh jobmanager &
sleep 5
exec flink run $@
else
exec /docker-entrypoint.sh $@
fi

View File

@ -7,21 +7,24 @@ version: "3.6"
services:
##flink
jobmanager:
build: Dockerfile
container_name: "${PINPOINT_FLINK_NAME}-jobmanager"
image: flink:1.3.1
image: ${PINPOINT_FLINK_NAME}-flink:dev
expose:
- "6123"
ports:
- "${FLINK_WEB_PORT:-8081}:8081"
command: jobmanager
command: standalone-job -p 1 pinpoint-flink-job.jar -spring.profiles.active release
environment:
- JOB_MANAGER_RPC_ADDRESS=jobmanager
- PINPOINT_ZOOKEEPER_ADDRESS=${PINPOINT_ZOOKEEPER_ADDRESS}
networks:
- pinpoint
taskmanager:
build: Dockerfile
container_name: "${PINPOINT_FLINK_NAME}-taskmanager"
image: flink:1.3.1
image: ${PINPOINT_FLINK_NAME}-flink:dev
expose:
- "6121"
- "6122"