java-Apache-Flink-External Jar


0

我尝试使用maven和一个额外的依赖项创建一个Flink应用程序:

 <dependency>
      <groupId>com.microsoft.sqlserver</groupId>
      <artifactId>mssql-jdbc</artifactId>
      <version>8.2.2.jre11</version>
  </dependency>

我正试图从我们的数据库中获取额外的信息。当我在 local运行代码时,一切正常,但在Flink上运行时,我得到以下错误消息:

所以我最初的想法是,使用maven shadow插件并没有将依赖项加载到fat jar上,但我认为是这样。

我的燧石在码头集装箱里运行,我在集装箱上打开1433:1433港口(在合成中)。

我的目标是将SQL数据添加到带有用户自定义规则的警报系统的流中。

这是我的pom.xml文件:

-->

<groupId>org.qscale</groupId>
<artifactId>FlinkPrototype</artifactId>
<version>1.0</version>
<packaging>jar</packaging>

<name>Flink Quickstart Job</name>
<url>http://www.myorganization.org</url>

<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<flink.version>1.10.0</flink.version>
<java.version>1.8</java.version>
<scala.binary.version>2.11</scala.binary.version>
<maven.compiler.source>${java.version}</maven.compiler.source>
<maven.compiler.target>${java.version}</maven.compiler.target>
</properties>

<repositories>
<repository>
<id>apache.snapshots</id>
<name>Apache Development Snapshot Repository</name>
<url>https://repository.apache.org/content/repositories/snapshots/</url>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</repository>
</repositories>

<dependencies>
<!-- Apache Flink dependencies -->
<!-- These dependencies are provided, because they should not be packaged into the JAR file. -->
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-java</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java_${scala.binary.version}</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>

&lt;!-- Add connector dependencies here. They must be in the default scope (compile). --&gt;

&lt;!-- Example:

&lt;dependency&gt;
    &lt;groupId&gt;org.apache.flink&lt;/groupId&gt;
    &lt;artifactId&gt;flink-connector-kafka-0.10_${scala.binary.version}&lt;/artifactId&gt;
    &lt;version&gt;${flink.version}&lt;/version&gt;
&lt;/dependency&gt;
--&gt;

&lt;!-- Project dependencies --&gt;
&lt;dependency&gt;
    &lt;groupId&gt;com.microsoft.sqlserver&lt;/groupId&gt;
    &lt;artifactId&gt;mssql-jdbc&lt;/artifactId&gt;
    &lt;version&gt;8.2.2.jre11&lt;/version&gt;
&lt;/dependency&gt;

&lt;!-- Add logging framework, to produce console output when running in the IDE. --&gt;
&lt;!-- These dependencies are excluded from the application JAR by default. --&gt;
&lt;dependency&gt;
    &lt;groupId&gt;org.slf4j&lt;/groupId&gt;
    &lt;artifactId&gt;slf4j-log4j12&lt;/artifactId&gt;
    &lt;version&gt;1.7.7&lt;/version&gt;
    &lt;scope&gt;runtime&lt;/scope&gt;
&lt;/dependency&gt;
&lt;dependency&gt;
    &lt;groupId&gt;log4j&lt;/groupId&gt;
    &lt;artifactId&gt;log4j&lt;/artifactId&gt;
    &lt;version&gt;1.2.17&lt;/version&gt;
    &lt;scope&gt;runtime&lt;/scope&gt;
&lt;/dependency&gt;

</dependencies>

<build>
<plugins>

    &lt;!-- Java Compiler --&gt;
    &lt;plugin&gt;
        &lt;groupId&gt;org.apache.maven.plugins&lt;/groupId&gt;
        &lt;artifactId&gt;maven-compiler-plugin&lt;/artifactId&gt;
        &lt;version&gt;3.1&lt;/version&gt;
        &lt;configuration&gt;
            &lt;source&gt;${java.version}&lt;/source&gt;
            &lt;target&gt;${java.version}&lt;/target&gt;
        &lt;/configuration&gt;
    &lt;/plugin&gt;

    &lt;!-- We use the maven-shade plugin to create a fat jar that contains all necessary dependencies. --&gt;
    &lt;!-- Change the value of &lt;mainClass&gt;...&lt;/mainClass&gt; if your program entry point changes. --&gt;
    &lt;plugin&gt;
        &lt;groupId&gt;org.apache.maven.plugins&lt;/groupId&gt;
        &lt;artifactId&gt;maven-shade-plugin&lt;/artifactId&gt;
        &lt;version&gt;3.1.1&lt;/version&gt;
        &lt;executions&gt;
            &lt;!-- Run shade goal on package phase --&gt;
            &lt;execution&gt;
                &lt;phase&gt;package&lt;/phase&gt;
                &lt;goals&gt;
                    &lt;goal&gt;shade&lt;/goal&gt;
                &lt;/goals&gt;
                &lt;configuration&gt;
                    &lt;shadedArtifactAttached&gt;true&lt;/shadedArtifactAttached&gt;
                    &lt;shadedClassifierName&gt;shaded&lt;/shadedClassifierName&gt;
                    &lt;artifactSet&gt;
                        &lt;excludes&gt;
                            &lt;exclude&gt;org.apache.flink:force-shading&lt;/exclude&gt;
                            &lt;exclude&gt;com.google.code.findbugs:jsr305&lt;/exclude&gt;
                            &lt;exclude&gt;org.slf4j:*&lt;/exclude&gt;
                            &lt;exclude&gt;log4j:*&lt;/exclude&gt;
                        &lt;/excludes&gt;
                    &lt;/artifactSet&gt;
                    &lt;filters&gt;
                        &lt;filter&gt;
                            &lt;!-- Do not copy the signatures in the META-INF folder.
                            Otherwise, this might cause SecurityExceptions when using the JAR. --&gt;
                            &lt;artifact&gt;*:*&lt;/artifact&gt;
                            &lt;excludes&gt;
                                &lt;exclude&gt;META-INF/*.SF&lt;/exclude&gt;
                                &lt;exclude&gt;META-INF/*.DSA&lt;/exclude&gt;
                                &lt;exclude&gt;META-INF/*.RSA&lt;/exclude&gt;
                            &lt;/excludes&gt;
                        &lt;/filter&gt;
                    &lt;/filters&gt;
                    &lt;transformers&gt;
                        &lt;transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"&gt;
                            &lt;mainClass&gt;org.qscale.StreamingJob&lt;/mainClass&gt;
                        &lt;/transformer&gt;
                    &lt;/transformers&gt;
                &lt;/configuration&gt;
            &lt;/execution&gt;
        &lt;/executions&gt;
    &lt;/plugin&gt;
&lt;/plugins&gt;

&lt;pluginManagement&gt;
    &lt;plugins&gt;

        &lt;!-- This improves the out-of-the-box experience in Eclipse by resolving some warnings. --&gt;
        &lt;plugin&gt;
            &lt;groupId&gt;org.eclipse.m2e&lt;/groupId&gt;
            &lt;artifactId&gt;lifecycle-mapping&lt;/artifactId&gt;
            &lt;version&gt;1.0.0&lt;/version&gt;
            &lt;configuration&gt;
                &lt;lifecycleMappingMetadata&gt;
                    &lt;pluginExecutions&gt;
                        &lt;pluginExecution&gt;
                            &lt;pluginExecutionFilter&gt;
                                &lt;groupId&gt;org.apache.maven.plugins&lt;/groupId&gt;
                                &lt;artifactId&gt;maven-shade-plugin&lt;/artifactId&gt;
                                &lt;versionRange&gt;[3.1.1,)&lt;/versionRange&gt;
                                &lt;goals&gt;
                                    &lt;goal&gt;shade&lt;/goal&gt;
                                &lt;/goals&gt;
                            &lt;/pluginExecutionFilter&gt;
                            &lt;action&gt;
                                &lt;ignore/&gt;
                            &lt;/action&gt;
                        &lt;/pluginExecution&gt;
                        &lt;pluginExecution&gt;
                            &lt;pluginExecutionFilter&gt;
                                &lt;groupId&gt;org.apache.maven.plugins&lt;/groupId&gt;
                                &lt;artifactId&gt;maven-compiler-plugin&lt;/artifactId&gt;
                                &lt;versionRange&gt;[3.1,)&lt;/versionRange&gt;
                                &lt;goals&gt;
                                    &lt;goal&gt;testCompile&lt;/goal&gt;
                                    &lt;goal&gt;compile&lt;/goal&gt;
                                &lt;/goals&gt;
                            &lt;/pluginExecutionFilter&gt;
                            &lt;action&gt;
                                &lt;ignore/&gt;
                            &lt;/action&gt;
                        &lt;/pluginExecution&gt;
                    &lt;/pluginExecutions&gt;
                &lt;/lifecycleMappingMetadata&gt;
            &lt;/configuration&gt;
        &lt;/plugin&gt;
    &lt;/plugins&gt;
&lt;/pluginManagement&gt;

</build>

谢谢你的帮助!

2 答案


0

添加类forName("com.microsoft.sqlserver.jdbc.SQLServerDriver“)在你的main方法中会对你有用,因为阴影看起来是正确的。

另一个问题是在pom中使用java1.8进行编译,但是添加了一个用java11编译的依赖项。将jdbc版本更改为以下版本将有望奏效。

<dependency>
      <groupId>com.microsoft.sqlserver</groupId>
      <artifactId>mssql-jdbc</artifactId>
      <version>8.2.2.jre8</version>
  </dependency>

更多细节请 读取


0

你怎么在 Flink上运行的?在IDE中?码头 worker?EC2型

您还可以尝试更改pom中要编译的范围依赖项。

编译


我来回答

写文章

提问题

面试题