java—使用Docker构建一个Maven项目并作为Flink job提交


0

我正在尝试使用docker构建一个maven项目,并在flink中提交一个生成的jar。但最终,一些例外情况正在发生。任何人都可以帮我解决这个例外。

Dockerfile文件

FROM maven:3.6-jdk-8-slim AS builder

COPY src /home/app/src
COPY pom.xml /home/app

RUN mvn -f /home/app/pom.xml clean package

FROM flink:1.10.0-scala_2.12
COPY --from=builder /home/app/target/demo-1.0-SNAPSHOT-jar-with-dependencies.jar /usr/local/lib/demo.jar

ENV FLINK_HOME=/opt/flink
ENV PATH=$FLINK_HOME/bin:$PATH

WORKDIR $FLINK_HOME

EXPOSE 8081

ENTRYPOINT ["./bin/flink", "run", "-c", "com.demo.app.App", "/usr/local/lib/demo.jar"]

码头 worker-合成.yml

version: "3"
services:
  demo:
    build:
      context: .
      dockerfile: Dockerfile
    container_name: demoapp
    expose:
      - "6121"
      - "6122"
      - "6123"
    ports:
      - "8081:8081"

例外

------------------------------------------------------------
Building demo
Step 1/10 : FROM maven:3.6-jdk-8-slim AS builder
 ---> addee4586ff4
Step 2/10 : COPY src /home/app/src
 ---> Using cache
 ---> 24e9837c9bc5
Step 3/10 : COPY pom.xml /home/app
 ---> Using cache
 ---> 13965367c201
Step 4/10 : RUN mvn -f /home/app/pom.xml clean package
 ---> Using cache
 ---> 0a6dec38b357
Step 5/10 : FROM flink:1.10.0-scala_2.12
 ---> d373f7bf2f75
Step 6/10 : COPY --from=builder /home/app/target/demo-1.0-SNAPSHOT-jar-with-dependencies.jar /usr/local/lib/demo.jar
 ---> 431c3080d639
Step 7/10 : ENV FLINK_HOME=/opt/flink
 ---> Running in bbae40bc5d84
Removing intermediate container bbae40bc5d84
 ---> 59171a4fd11e
Step 8/10 : ENV PATH=$FLINK_HOME/bin:$PATH
 ---> Running in 45e30a54e742
Removing intermediate container 45e30a54e742
 ---> 4e986527391a
Step 9/10 : WORKDIR $FLINK_HOME
 ---> Running in 1f0ca2067595
Removing intermediate container 1f0ca2067595
 ---> aa817f1dee2d
Step 10/10 : ENTRYPOINT ["./bin/flink", "run", "-c", "com.demo.app.App", "/usr/local/lib/demo.jar"]
 ---> Running in f984981dfbc8
Removing intermediate container f984981dfbc8
 ---> 1cdc22379a90
Successfully built 1cdc22379a90
Successfully tagged demo:latest
Creating demo ... 
Creating demo ... done
Attaching to demo
demo | 
demo | ------------------------------------------------------------
demo |  The program finished with the following exception:
demo | 
demo | org.apache.flink.client.program.ProgramInvocationException: The main method caused an error: java.util.concurrent.ExecutionException: org.apache.flink.runtime.client.JobSubmissionException: Failed to submit JobGraph.
demo |  at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:335)
demo |  at org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:205)
demo |  at org.apache.flink.client.ClientUtils.executeProgram(ClientUtils.java:138)
demo |  at org.apache.flink.client.cli.CliFrontend.executeProgram(CliFrontend.java:664)
demo |  at org.apache.flink.client.cli.CliFrontend.run(CliFrontend.java:213)
demo |  at org.apache.flink.client.cli.CliFrontend.parseParameters(CliFrontend.java:895)
demo |  at org.apache.flink.client.cli.CliFrontend.lambda$main$10(CliFrontend.java:968)
demo |  at org.apache.flink.runtime.security.NoOpSecurityContext.runSecured(NoOpSecurityContext.java:30)
demo |  at org.apache.flink.client.cli.CliFrontend.main(CliFrontend.java:968)
demo | Caused by: java.lang.RuntimeException: java.util.concurrent.ExecutionException: org.apache.flink.runtime.client.JobSubmissionException: Failed to submit JobGraph.
demo |  at org.apache.flink.util.ExceptionUtils.rethrow(ExceptionUtils.java:199)
demo |  at org.apache.flink.streaming.api.environment.StreamExecutionEnvironment.executeAsync(StreamExecutionEnvironment.java:1741)
demo |  at org.apache.flink.streaming.api.environment.StreamContextEnvironment.executeAsync(StreamContextEnvironment.java:94)
demo |  at org.apache.flink.streaming.api.environment.StreamContextEnvironment.execute(StreamContextEnvironment.java:63)
demo |  at org.apache.flink.streaming.api.environment.StreamExecutionEnvironment.execute(StreamExecutionEnvironment.java:1620)
demo |  at com.demo.app.App.main(App.java:105)
demo |  at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
demo |  at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
demo |  at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
demo |  at java.lang.reflect.Method.invoke(Method.java:498)
demo |  at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:321)
demo |  ... 8 more
demo | Caused by: java.util.concurrent.ExecutionException: org.apache.flink.runtime.client.JobSubmissionException: Failed to submit JobGraph.
demo |  at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:357)
demo |  at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1908)
demo |  at org.apache.flink.streaming.api.environment.StreamExecutionEnvironment.executeAsync(StreamExecutionEnvironment.java:1736)
demo |  ... 17 more
demo | Caused by: org.apache.flink.runtime.client.JobSubmissionException: Failed to submit JobGraph.
demo |  at org.apache.flink.client.program.rest.RestClusterClient.lambda$submitJob$7(RestClusterClient.java:359)
demo |  at java.util.concurrent.CompletableFuture.uniExceptionally(CompletableFuture.java:884)
demo |  at java.util.concurrent.CompletableFuture$UniExceptionally.tryFire(CompletableFuture.java:866)
demo |  at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488)
demo |  at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1990)
demo |  at org.apache.flink.runtime.concurrent.FutureUtils.lambda$retryOperationWithDelay$8(FutureUtils.java:287)
demo |  at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:774)
demo |  at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:750)
demo |  at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:488)
demo |  at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:1990)
demo |  at org.apache.flink.runtime.rest.RestClient.lambda$submitRequest$1(RestClient.java:342)
demo |  at org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:500)
demo |  at org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:493)
demo |  at org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:472)
demo |  at org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:413)
demo |  at org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:538)
demo |  at org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.setFailure0(DefaultPromise.java:531)
demo |  at org.apache.flink.shaded.netty4.io.netty.util.concurrent.DefaultPromise.tryFailure(DefaultPromise.java:111)
demo |  at org.apache.flink.shaded.netty4.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.fulfillConnectPromise(AbstractNioChannel.java:323)
demo |  at org.apache.flink.shaded.netty4.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339)
demo |  at org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:685)
demo |  at org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:632)
demo |  at org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:549)
demo |  at org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:511)
demo |  at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:918)
demo |  at org.apache.flink.shaded.netty4.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
demo |  at java.lang.Thread.run(Thread.java:748)
demo | Caused by: org.apache.flink.runtime.concurrent.FutureUtils$RetryException: Could not complete the operation. Number of retries has been exhausted.
demo |  at org.apache.flink.runtime.concurrent.FutureUtils.lambda$retryOperationWithDelay$8(FutureUtils.java:284)
demo |  ... 21 more
demo | Caused by: java.util.concurrent.CompletionException: org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: localhost/127.0.0.1:8081
demo |  at java.util.concurrent.CompletableFuture.encodeThrowable(CompletableFuture.java:292)
demo |  at java.util.concurrent.CompletableFuture.completeThrowable(CompletableFuture.java:308)
demo |  at java.util.concurrent.CompletableFuture.uniCompose(CompletableFuture.java:957)
demo |  at java.util.concurrent.CompletableFuture$UniCompose.tryFire(CompletableFuture.java:940)
demo |  ... 19 more
demo | Caused by: org.apache.flink.shaded.netty4.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: localhost/127.0.0.1:8081
demo | Caused by: java.net.ConnectException: Connection refused
demo |  at sun.nio.ch.SocketChannelImpl.checkConnect(Native Method)
demo |  at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:714)
demo |  at org.apache.flink.shaded.netty4.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:327)
demo |  at org.apache.flink.shaded.netty4.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:336)
demo |  at org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:685)
demo |  at org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:632)
demo |  at org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:549)
demo |  at org.apache.flink.shaded.netty4.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:511)
demo |  at org.apache.flink.shaded.netty4.io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:918)
demo |  at org.apache.flink.shaded.netty4.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)
demo |  at java.lang.Thread.run(Thread.java:748)

另外,检查了正在运行的容器,但没有运行任何容器。

我使用的构建命令

sudo docker-compose up --build

也尝试使用此命令生成

docker run --rm -it demo:latest
sudo docker build -t demo .

但结果是一样的例外

1 答案


0

这不是怎么和多克一起玩Flink的。您需要安排启动一个 job管理器和至少一个任务管理器——在没有为 job运行创建集群之前,不能使用./bin/flink run。

了解这个主题的一个好方法是通过Flink Operations Playground进行学习,其中包括一个完整的基于docker的基础设施,您可以将其用作自己项目的起点。

还有关于如何在Docker上运行Flink的文档。


我来回答

写文章

提问题

面试题