Skip to content

Commit

Permalink
Build: Append _2.12 (scala version) to Spark 3.1 and 3.2 module names (…
Browse files Browse the repository at this point in the history 
…apache#3624)
  • Loading branch information
@kbendick
kbendick authored Nov 30, 2021
1 parent 466073b commit 4f6b67f
Show file tree
Hide file tree
Showing 5 changed files with 48 additions and 48 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/spark-ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ jobs:
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle') }}
restore-keys: ${{ runner.os }}-gradle
- run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
- run: ./gradlew -DsparkVersions=${{ matrix.spark }} -DhiveVersions= -DflinkVersions= :iceberg-spark:iceberg-spark-${{ matrix.spark }}:check :iceberg-spark:iceberg-spark-${{ matrix.spark }}-extensions:check :iceberg-spark:iceberg-spark-${{ matrix.spark }}-runtime:check -Pquick=true -x javadoc
- run: ./gradlew -DsparkVersions=${{ matrix.spark }} -DhiveVersions= -DflinkVersions= :iceberg-spark:iceberg-spark-${{ matrix.spark }}_2.12:check :iceberg-spark:iceberg-spark-${{ matrix.spark }}-extensions_2.12:check :iceberg-spark:iceberg-spark-${{ matrix.spark }}-runtime_2.12:check -Pquick=true -x javadoc
- uses: actions/upload-artifact@v2
if: failure()
with:
Expand Down
2 changes: 1 addition & 1 deletion jmh.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ if (sparkVersions.contains("3.0")) {
}

if (sparkVersions.contains("3.2")) {
jmhProjects.add(project(":iceberg-spark:iceberg-spark-3.2"))
jmhProjects.add(project(":iceberg-spark:iceberg-spark-3.2_2.12"))
}

configure(jmhProjects) {
Expand Down
36 changes: 18 additions & 18 deletions settings.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,27 +108,27 @@ if (sparkVersions.contains("3.0")) {
}

if (sparkVersions.contains("3.1")) {
include ':iceberg-spark:spark-3.1'
include ':iceberg-spark:spark-3.1-extensions'
include ':iceberg-spark:spark-3.1-runtime'
project(':iceberg-spark:spark-3.1').projectDir = file('spark/v3.1/spark')
project(':iceberg-spark:spark-3.1').name = 'iceberg-spark-3.1'
project(':iceberg-spark:spark-3.1-extensions').projectDir = file('spark/v3.1/spark-extensions')
project(':iceberg-spark:spark-3.1-extensions').name = 'iceberg-spark-3.1-extensions'
project(':iceberg-spark:spark-3.1-runtime').projectDir = file('spark/v3.1/spark-runtime')
project(':iceberg-spark:spark-3.1-runtime').name = 'iceberg-spark-3.1-runtime'
include ':iceberg-spark:spark-3.1_2.12'
include ':iceberg-spark:spark-3.1-extensions_2.12'
include ':iceberg-spark:spark-3.1-runtime_2.12'
project(':iceberg-spark:spark-3.1_2.12').projectDir = file('spark/v3.1/spark')
project(':iceberg-spark:spark-3.1_2.12').name = 'iceberg-spark-3.1_2.12'
project(':iceberg-spark:spark-3.1-extensions_2.12').projectDir = file('spark/v3.1/spark-extensions')
project(':iceberg-spark:spark-3.1-extensions_2.12').name = 'iceberg-spark-3.1-extensions_2.12'
project(':iceberg-spark:spark-3.1-runtime_2.12').projectDir = file('spark/v3.1/spark-runtime')
project(':iceberg-spark:spark-3.1-runtime_2.12').name = 'iceberg-spark-3.1-runtime_2.12'
}

if (sparkVersions.contains("3.2")) {
include ':iceberg-spark:spark-3.2'
include ':iceberg-spark:spark-3.2-extensions'
include ':iceberg-spark:spark-3.2-runtime'
project(':iceberg-spark:spark-3.2').projectDir = file('spark/v3.2/spark')
project(':iceberg-spark:spark-3.2').name = 'iceberg-spark-3.2'
project(':iceberg-spark:spark-3.2-extensions').projectDir = file('spark/v3.2/spark-extensions')
project(':iceberg-spark:spark-3.2-extensions').name = 'iceberg-spark-3.2-extensions'
project(':iceberg-spark:spark-3.2-runtime').projectDir = file('spark/v3.2/spark-runtime')
project(':iceberg-spark:spark-3.2-runtime').name = 'iceberg-spark-3.2-runtime'
include ':iceberg-spark:spark-3.2_2.12'
include ':iceberg-spark:spark-3.2-extensions_2.12'
include ':iceberg-spark:spark-3.2-runtime_2.12'
project(':iceberg-spark:spark-3.2_2.12').projectDir = file('spark/v3.2/spark')
project(':iceberg-spark:spark-3.2_2.12').name = 'iceberg-spark-3.2_2.12'
project(':iceberg-spark:spark-3.2-extensions_2.12').projectDir = file('spark/v3.2/spark-extensions')
project(':iceberg-spark:spark-3.2-extensions_2.12').name = 'iceberg-spark-3.2-extensions_2.12'
project(':iceberg-spark:spark-3.2-runtime_2.12').projectDir = file('spark/v3.2/spark-runtime')
project(':iceberg-spark:spark-3.2-runtime_2.12').name = 'iceberg-spark-3.2-runtime_2.12'
}

// hive 3 depends on hive 2, so always add hive 2 if hive3 is enabled
Expand Down
28 changes: 14 additions & 14 deletions spark/v3.1/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@
*/

def sparkProjects = [
project(':iceberg-spark:iceberg-spark-3.1'),
project(":iceberg-spark:iceberg-spark-3.1-extensions"),
project(':iceberg-spark:iceberg-spark-3.1-runtime')
project(':iceberg-spark:iceberg-spark-3.1_2.12'),
project(":iceberg-spark:iceberg-spark-3.1-extensions_2.12"),
project(':iceberg-spark:iceberg-spark-3.1-runtime_2.12')
]

configure(sparkProjects) {
Expand All @@ -38,7 +38,7 @@ configure(sparkProjects) {
}
}

project(':iceberg-spark:iceberg-spark-3.1') {
project(':iceberg-spark:iceberg-spark-3.1_2.12') {
apply plugin: 'scala'

sourceSets {
Expand Down Expand Up @@ -104,7 +104,7 @@ project(':iceberg-spark:iceberg-spark-3.1') {
}
}

project(":iceberg-spark:iceberg-spark-3.1-extensions") {
project(":iceberg-spark:iceberg-spark-3.1-extensions_2.12") {
apply plugin: 'java-library'
apply plugin: 'scala'
apply plugin: 'antlr'
Expand All @@ -130,7 +130,7 @@ project(":iceberg-spark:iceberg-spark-3.1-extensions") {
compileOnly project(':iceberg-data')
compileOnly project(':iceberg-orc')
compileOnly project(':iceberg-common')
compileOnly project(':iceberg-spark:iceberg-spark-3.1')
compileOnly project(':iceberg-spark:iceberg-spark-3.1_2.12')
compileOnly project(':iceberg-hive-metastore')
compileOnly("org.apache.spark:spark-hive_2.12:${sparkVersion}") {
exclude group: 'org.apache.avro', module: 'avro'
Expand All @@ -144,7 +144,7 @@ project(":iceberg-spark:iceberg-spark-3.1-extensions") {
testImplementation project(path: ':iceberg-orc', configuration: 'testArtifacts')
testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
testImplementation project(path: ':iceberg-spark:iceberg-spark-3.1', configuration: 'testArtifacts')
testImplementation project(path: ':iceberg-spark:iceberg-spark-3.1_2.12', configuration: 'testArtifacts')

testImplementation "org.apache.avro:avro"

Expand All @@ -160,7 +160,7 @@ project(":iceberg-spark:iceberg-spark-3.1-extensions") {
}
}

project(':iceberg-spark:iceberg-spark-3.1-runtime') {
project(':iceberg-spark:iceberg-spark-3.1-runtime_2.12') {
apply plugin: 'com.github.johnrengelman.shadow'

tasks.jar.dependsOn tasks.shadowJar
Expand Down Expand Up @@ -194,8 +194,8 @@ project(':iceberg-spark:iceberg-spark-3.1-runtime') {

dependencies {
api project(':iceberg-api')
implementation project(':iceberg-spark:iceberg-spark-3.1')
implementation project(':iceberg-spark:iceberg-spark-3.1-extensions')
implementation project(':iceberg-spark:iceberg-spark-3.1_2.12')
implementation project(':iceberg-spark:iceberg-spark-3.1-extensions_2.12')
implementation project(':iceberg-aws')
implementation(project(':iceberg-nessie')) {
exclude group: 'com.google.code.findbugs', module: 'jsr305'
Expand All @@ -206,11 +206,11 @@ project(':iceberg-spark:iceberg-spark-3.1-runtime') {
integrationImplementation 'org.slf4j:slf4j-simple'
integrationImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
integrationImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
integrationImplementation project(path: ':iceberg-spark:iceberg-spark-3.1', configuration: 'testArtifacts')
integrationImplementation project(path: ':iceberg-spark:iceberg-spark-3.1-extensions', configuration: 'testArtifacts')
integrationImplementation project(path: ':iceberg-spark:iceberg-spark-3.1_2.12', configuration: 'testArtifacts')
integrationImplementation project(path: ':iceberg-spark:iceberg-spark-3.1-extensions_2.12', configuration: 'testArtifacts')
// Not allowed on our classpath, only the runtime jar is allowed
integrationCompileOnly project(':iceberg-spark:iceberg-spark-3.1-extensions')
integrationCompileOnly project(':iceberg-spark:iceberg-spark-3.1')
integrationCompileOnly project(':iceberg-spark:iceberg-spark-3.1-extensions_2.12')
integrationCompileOnly project(':iceberg-spark:iceberg-spark-3.1_2.12')
integrationCompileOnly project(':iceberg-api')
}

Expand Down
28 changes: 14 additions & 14 deletions spark/v3.2/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@
*/

def sparkProjects = [
project(':iceberg-spark:iceberg-spark-3.2'),
project(":iceberg-spark:iceberg-spark-3.2-extensions"),
project(':iceberg-spark:iceberg-spark-3.2-runtime')
project(':iceberg-spark:iceberg-spark-3.2_2.12'),
project(":iceberg-spark:iceberg-spark-3.2-extensions_2.12"),
project(':iceberg-spark:iceberg-spark-3.2-runtime_2.12')
]

configure(sparkProjects) {
Expand All @@ -38,7 +38,7 @@ configure(sparkProjects) {
}
}

project(':iceberg-spark:iceberg-spark-3.2') {
project(':iceberg-spark:iceberg-spark-3.2_2.12') {
apply plugin: 'scala'

sourceSets {
Expand Down Expand Up @@ -108,7 +108,7 @@ project(':iceberg-spark:iceberg-spark-3.2') {
}
}

project(":iceberg-spark:iceberg-spark-3.2-extensions") {
project(":iceberg-spark:iceberg-spark-3.2-extensions_2.12") {
apply plugin: 'java-library'
apply plugin: 'scala'
apply plugin: 'antlr'
Expand All @@ -132,7 +132,7 @@ project(":iceberg-spark:iceberg-spark-3.2-extensions") {
compileOnly project(':iceberg-api')
compileOnly project(':iceberg-core')
compileOnly project(':iceberg-common')
compileOnly project(':iceberg-spark:iceberg-spark-3.2')
compileOnly project(':iceberg-spark:iceberg-spark-3.2_2.12')
compileOnly project(':iceberg-hive-metastore')
compileOnly("org.apache.spark:spark-hive_2.12:${sparkVersion}") {
exclude group: 'org.apache.avro', module: 'avro'
Expand All @@ -146,7 +146,7 @@ project(":iceberg-spark:iceberg-spark-3.2-extensions") {

testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
testImplementation project(path: ':iceberg-spark:iceberg-spark-3.2', configuration: 'testArtifacts')
testImplementation project(path: ':iceberg-spark:iceberg-spark-3.2_2.12', configuration: 'testArtifacts')

testImplementation "org.apache.avro:avro"

Expand All @@ -162,7 +162,7 @@ project(":iceberg-spark:iceberg-spark-3.2-extensions") {
}
}

project(':iceberg-spark:iceberg-spark-3.2-runtime') {
project(':iceberg-spark:iceberg-spark-3.2-runtime_2.12') {
apply plugin: 'com.github.johnrengelman.shadow'

tasks.jar.dependsOn tasks.shadowJar
Expand Down Expand Up @@ -196,8 +196,8 @@ project(':iceberg-spark:iceberg-spark-3.2-runtime') {

dependencies {
api project(':iceberg-api')
implementation project(':iceberg-spark:iceberg-spark-3.2')
implementation project(':iceberg-spark:iceberg-spark-3.2-extensions')
implementation project(':iceberg-spark:iceberg-spark-3.2_2.12')
implementation project(':iceberg-spark:iceberg-spark-3.2-extensions_2.12')
implementation project(':iceberg-aws')
implementation(project(':iceberg-nessie')) {
exclude group: 'com.google.code.findbugs', module: 'jsr305'
Expand All @@ -208,11 +208,11 @@ project(':iceberg-spark:iceberg-spark-3.2-runtime') {
integrationImplementation 'org.slf4j:slf4j-simple'
integrationImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
integrationImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
integrationImplementation project(path: ':iceberg-spark:iceberg-spark-3.2', configuration: 'testArtifacts')
integrationImplementation project(path: ':iceberg-spark:iceberg-spark-3.2-extensions', configuration: 'testArtifacts')
integrationImplementation project(path: ':iceberg-spark:iceberg-spark-3.2_2.12', configuration: 'testArtifacts')
integrationImplementation project(path: ':iceberg-spark:iceberg-spark-3.2-extensions_2.12', configuration: 'testArtifacts')
// Not allowed on our classpath, only the runtime jar is allowed
integrationCompileOnly project(':iceberg-spark:iceberg-spark-3.2-extensions')
integrationCompileOnly project(':iceberg-spark:iceberg-spark-3.2')
integrationCompileOnly project(':iceberg-spark:iceberg-spark-3.2-extensions_2.12')
integrationCompileOnly project(':iceberg-spark:iceberg-spark-3.2_2.12')
integrationCompileOnly project(':iceberg-api')
}

Expand Down

0 comments on commit 4f6b67f

Please sign in to comment.