Skip to content

Commit

Permalink
[ZEPPELIN-5972] Support Flink 1.17 (#4677)
Browse files Browse the repository at this point in the history
* Support Flink 1.17

* Bump Flink 1.17.1

* Fix package name

* fix java 11 compile

* empty

---------

Co-authored-by: Jeff Zhang <zjffdu@apache.org>
  • Loading branch information
pan3793 and zjffdu authored Oct 22, 2023
1 parent 9680241 commit efadcd6
Show file tree
Hide file tree
Showing 13 changed files with 1,842 additions and 5 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/core.yml
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,7 @@ jobs:
strategy:
fail-fast: false
matrix:
flink: [113, 114, 115, 116]
flink: [113, 114, 115, 116, 117]
steps:
- name: Checkout
uses: actions/checkout@v3
Expand Down
42 changes: 42 additions & 0 deletions flink/flink-scala-parent/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,12 @@
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>org.apache.zeppelin</groupId>
<artifactId>flink1.17-shims</artifactId>
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>org.apache.zeppelin</groupId>
<artifactId>zeppelin-python</artifactId>
Expand Down Expand Up @@ -1007,6 +1013,42 @@
</dependencies>
</profile>

<profile>
<id>flink-117</id>
<properties>
<flink.version>${flink1.17.version}</flink.version>
<flink.scala.version>2.12.7</flink.scala.version>
<flink.scala.binary.version>2.12</flink.scala.binary.version>
<flink.library.scala.suffix></flink.library.scala.suffix>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-runtime</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-planner_${flink.scala.binary.version}</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-sql-client</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-python</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>
</dependencies>
</profile>

<profile>
<id>hive2</id>
<activation>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -511,6 +511,11 @@ public void testCatalog() throws IOException, InterpreterException{
resultMessages = context.out.toInterpreterResultMessage();
assertEquals("current catalog: test_catalog\n", resultMessages.get(0).getData());

// USE DEFAULT_CATALOG
context = getInterpreterContext();
result = sqlInterpreter.interpret("USE CATALOG default_catalog", context);
assertEquals(context.out.toString(), InterpreterResult.Code.SUCCESS, result.code());

// DROP CATALOG
context = getInterpreterContext();
result = sqlInterpreter.interpret(
Expand All @@ -526,8 +531,6 @@ public void testCatalog() throws IOException, InterpreterException{
assertEquals(context.out.toString(), InterpreterResult.Code.SUCCESS, result.code());
resultMessages = context.out.toInterpreterResultMessage();
assertTrue(context.out.toString(), resultMessages.get(0).getData().contains("default_catalog"));
assertFalse(context.out.toString(), resultMessages.get(0).getData().contains("test_catalog"));

}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,9 @@ private static FlinkShims loadShims(FlinkVersion flinkVersion,
} else if (flinkVersion.getMajorVersion() == 1 && flinkVersion.getMinorVersion() == 16) {
LOGGER.info("Initializing shims for Flink 1.16");
flinkShimsClass = Class.forName("org.apache.zeppelin.flink.Flink116Shims");
} else if (flinkVersion.getMajorVersion() == 1 && flinkVersion.getMinorVersion() == 17) {
LOGGER.info("Initializing shims for Flink 1.17");
flinkShimsClass = Class.forName("org.apache.zeppelin.flink.Flink117Shims");
} else {
throw new Exception("Flink version: '" + flinkVersion + "' is not supported yet");
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.flink.api.common.RuntimeExecutionMode;
import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.client.cli.CliFrontend;
Expand Down Expand Up @@ -53,7 +52,6 @@
import org.apache.flink.table.functions.TableAggregateFunction;
import org.apache.flink.table.functions.TableFunction;
import org.apache.flink.table.module.ModuleManager;
import org.apache.flink.table.planner.calcite.FlinkTypeFactory;
import org.apache.flink.table.resource.ResourceManager;
import org.apache.flink.table.sinks.TableSink;
import org.apache.flink.table.typeutils.TimeIndicatorTypeInfo;
Expand Down
207 changes: 207 additions & 0 deletions flink/flink1.17-shims/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,207 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>flink-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.11.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>

<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.zeppelin</groupId>
<artifactId>flink1.17-shims</artifactId>
<version>0.11.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Zeppelin: Flink1.17 Shims</name>

<properties>
<flink.version>${flink1.17.version}</flink.version>
<flink.scala.binary.version>2.12</flink.scala.binary.version>
</properties>

<dependencies>

<dependency>
<groupId>org.apache.zeppelin</groupId>
<artifactId>flink-shims</artifactId>
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-core</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-clients</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-runtime</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-api-scala_${flink.scala.binary.version}</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-api-scala-bridge_${flink.scala.binary.version}</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-api-java-bridge</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-scala_${flink.scala.binary.version}</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-scala_${flink.scala.binary.version}</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-java</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-planner_${flink.scala.binary.version}</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-python</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-sql-client</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>

</dependencies>

<build>
<plugins>

<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<executions>
<execution>
<id>eclipse-add-source</id>
<goals>
<goal>add-source</goal>
</goals>
</execution>
<execution>
<id>scala-compile-first</id>
<phase>process-resources</phase>
<goals>
<goal>compile</goal>
</goals>
</execution>
<execution>
<id>scala-test-compile-first</id>
<phase>process-test-resources</phase>
<goals>
<goal>testCompile</goal>
</goals>
</execution>
</executions>
<configuration>
<scalaVersion>${flink.scala.version}</scalaVersion>
<args>
<arg>-unchecked</arg>
<arg>-deprecation</arg>
<arg>-feature</arg>
<arg>-nobootcp</arg>
</args>
<jvmArgs>
<jvmArg>-Xms1024m</jvmArg>
<jvmArg>-Xmx1024m</jvmArg>
<jvmArg>-XX:MaxMetaspaceSize=${MaxMetaspace}</jvmArg>
</jvmArgs>
<javacArgs>
<javacArg>-source</javacArg>
<javacArg>${java.version}</javacArg>
<javacArg>-target</javacArg>
<javacArg>${java.version}</javacArg>
<javacArg>-Xlint:all,-serial,-path,-options</javacArg>
</javacArgs>
</configuration>
</plugin>

<plugin>
<artifactId>maven-resources-plugin</artifactId>
<executions>
<execution>
<id>copy-interpreter-setting</id>
<phase>none</phase>
<configuration>
<skip>true</skip>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>

</project>
Loading

0 comments on commit efadcd6

Please sign in to comment.