-
Notifications
You must be signed in to change notification settings - Fork 391
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[WIP] Scala 2.12 / Spark 3 upgrade #550
base: master
Are you sure you want to change the base?
Changes from 85 commits
f6264a7
685d6e1
e62772d
69247ac
330bf50
d6b0723
63b77b5
4e46e31
5a528e1
5f39603
0d1a0c0
0a4f906
660db62
3ecca64
507503a
348a392
f43cb26
4455034
a0978bf
82aa188
b27b47a
6535e4e
d1d7b9a
ac75e15
761b889
95095ed
76b411b
ecfb902
a1a2579
785ddc5
aacf00c
c0a888f
9ececc9
25a4449
53df597
4460fe5
5b29d8b
713a9f4
69a3678
f4b3f01
142f121
5ee32b1
6e8e130
c3ccdee
ae1dfcf
fd723d6
e0f0bd8
98dafde
accd2ba
27fdd3e
f0cbc9e
7fb9f0a
ff29d1b
20b8584
ca30345
e2078e1
fb16bd9
5b61508
807eca9
3fba576
dc4adbc
2cca254
d3fbf8f
9fbc9da
e8c5b7a
c61a5b7
017676a
0538892
3e252db
4fe2fdf
c1941e1
bdfae00
fe4f2fb
c649974
c391aac
5f55dd9
642d27c
1605bd4
64ea9d2
51806fd
09b2960
a946ffb
ec7da39
5b555e3
30e61a3
9363b20
6f7c841
9a04faf
4f752ab
6731b9d
b9e18ce
c42163d
7082707
355bbe2
2cb1827
fc5cdc8
dc014fa
f31ce9f
421b9bc
0038823
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -7,6 +7,7 @@ buildscript { | |
dependencies { | ||
classpath 'org.github.ngbinh.scalastyle:gradle-scalastyle-plugin_2.11:1.0.1' | ||
classpath 'com.commercehub.gradle.plugin:gradle-avro-plugin:0.16.0' | ||
classpath 'com.adtran:scala-multiversion-plugin:1.+' | ||
} | ||
} | ||
|
||
|
@@ -46,6 +47,7 @@ configure(allProjs) { | |
apply plugin: 'net.minecrell.licenser' | ||
apply plugin: 'com.github.jk1.dependency-license-report' | ||
apply plugin: 'com.github.johnrengelman.shadow' | ||
apply plugin: 'com.adtran.scala-multiversion-plugin' | ||
|
||
sourceCompatibility = 1.8 | ||
targetCompatibility = 1.8 | ||
|
@@ -54,23 +56,21 @@ configure(allProjs) { | |
mainClassName = "please.set.main.class.in.build.gradle" | ||
|
||
ext { | ||
scalaVersion = '2.11' | ||
scalaVersionRevision = '12' | ||
scalaTestVersion = '3.0.5' | ||
scalaCheckVersion = '1.14.0' | ||
junitVersion = '4.12' | ||
avroVersion = '1.8.2' | ||
sparkVersion = '2.4.5' | ||
sparkVersion = '3.1.1' | ||
scalaGraphVersion = '1.12.5' | ||
scalafmtVersion = '1.5.1' | ||
hadoopVersion = 'hadoop2' | ||
json4sVersion = '3.5.3' // matches Spark dependency version | ||
json4sVersion = '3.7.0-M5' // matches Spark dependency version | ||
jodaTimeVersion = '2.9.4' | ||
jodaConvertVersion = '1.8.1' | ||
algebirdVersion = '0.13.4' | ||
jacksonVersion = '2.7.3' | ||
jacksonVersion = '2.12.2' | ||
luceneVersion = '7.3.0' | ||
enumeratumVersion = '1.4.12' | ||
enumeratumVersion = '1.4.18' | ||
scoptVersion = '3.5.0' | ||
googleLibPhoneNumberVersion = '8.8.5' | ||
googleGeoCoderVersion = '2.82' | ||
|
@@ -80,15 +80,15 @@ configure(allProjs) { | |
collectionsVersion = '3.2.2' | ||
optimaizeLangDetectorVersion = '0.0.1' | ||
tikaVersion = '1.22' | ||
sparkTestingBaseVersion = '2.4.3_0.12.0' | ||
sparkTestingBaseVersion = '3.0.1_1.0.0' | ||
sourceCodeVersion = '0.1.3' | ||
pegdownVersion = '1.4.2' | ||
commonsValidatorVersion = '1.6' | ||
commonsIOVersion = '2.6' | ||
scoveragePluginVersion = '1.3.1' | ||
xgboostVersion = '0.90' | ||
akkaSlf4jVersion = '2.3.11' | ||
mleapVersion = '0.16.0' | ||
xgboostVersion = '1.3.1' | ||
akkaSlf4jVersion = '2.5.23' | ||
mleapVersion = '0.16.0' // TODO: upgrade to Spark 3-compatibel 0.17 when ready: https://github.com/combust/mleap/issues/727 | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. typo |
||
memoryFilesystemVersion = '2.1.0' | ||
} | ||
|
||
|
@@ -100,37 +100,37 @@ configure(allProjs) { | |
dependencies { | ||
// Scala | ||
zinc 'com.typesafe.zinc:zinc:0.3.15' | ||
scoverage "org.scoverage:scalac-scoverage-plugin_$scalaVersion:$scoveragePluginVersion" | ||
scoverage "org.scoverage:scalac-scoverage-runtime_$scalaVersion:$scoveragePluginVersion" | ||
scalaLibrary "org.scala-lang:scala-library:$scalaVersion.$scalaVersionRevision" | ||
scalaCompiler "org.scala-lang:scala-compiler:$scalaVersion.$scalaVersionRevision" | ||
compile "org.scala-lang:scala-library:$scalaVersion.$scalaVersionRevision" | ||
scoverage "org.scoverage:scalac-scoverage-plugin_%%:$scoveragePluginVersion" | ||
scoverage "org.scoverage:scalac-scoverage-runtime_%%:$scoveragePluginVersion" | ||
scalaLibrary "org.scala-lang:scala-library:$scalaVersion" | ||
scalaCompiler "org.scala-lang:scala-compiler:$scalaVersion" | ||
compile "org.scala-lang:scala-library:$scalaVersion" | ||
|
||
// Spark | ||
compileOnly "org.apache.spark:spark-core_$scalaVersion:$sparkVersion" | ||
testCompile "org.apache.spark:spark-core_$scalaVersion:$sparkVersion" | ||
compileOnly "org.apache.spark:spark-mllib_$scalaVersion:$sparkVersion" | ||
testCompile "org.apache.spark:spark-mllib_$scalaVersion:$sparkVersion" | ||
compileOnly "org.apache.spark:spark-sql_$scalaVersion:$sparkVersion" | ||
testCompile "org.apache.spark:spark-sql_$scalaVersion:$sparkVersion" | ||
compileOnly "org.apache.spark:spark-core_%%:$sparkVersion" | ||
testCompile "org.apache.spark:spark-core_%%:$sparkVersion" | ||
compileOnly "org.apache.spark:spark-mllib_%%:$sparkVersion" | ||
testCompile "org.apache.spark:spark-mllib_%%:$sparkVersion" | ||
compileOnly "org.apache.spark:spark-sql_%%:$sparkVersion" | ||
testCompile "org.apache.spark:spark-sql_%%:$sparkVersion" | ||
|
||
// Test | ||
compileOnly "org.scalatest:scalatest_$scalaVersion:$scalaTestVersion" | ||
testCompile "org.scalatest:scalatest_$scalaVersion:$scalaTestVersion" | ||
compileOnly "org.scalacheck:scalacheck_$scalaVersion:$scalaCheckVersion" | ||
testCompile "org.scoverage:scalac-scoverage-plugin_$scalaVersion:$scoveragePluginVersion" | ||
testCompile "org.scoverage:scalac-scoverage-runtime_$scalaVersion:$scoveragePluginVersion" | ||
testCompile "org.scalacheck:scalacheck_$scalaVersion:$scalaCheckVersion" | ||
testCompile ("com.holdenkarau:spark-testing-base_$scalaVersion:$sparkTestingBaseVersion") { transitive = false } | ||
compileOnly "org.scalatest:scalatest_%%:$scalaTestVersion" | ||
testCompile "org.scalatest:scalatest_%%:$scalaTestVersion" | ||
compileOnly "org.scalacheck:scalacheck_%%:$scalaCheckVersion" | ||
testCompile "org.scoverage:scalac-scoverage-plugin_%%:$scoveragePluginVersion" | ||
testCompile "org.scoverage:scalac-scoverage-runtime_%%:$scoveragePluginVersion" | ||
testCompile "org.scalacheck:scalacheck_%%:$scalaCheckVersion" | ||
testCompile ("com.holdenkarau:spark-testing-base_%%:$sparkTestingBaseVersion") { transitive = false } | ||
testCompile "junit:junit:$junitVersion" | ||
testRuntime "org.pegdown:pegdown:$pegdownVersion" | ||
} | ||
|
||
configurations.all { | ||
resolutionStrategy { | ||
force "commons-collections:commons-collections:$collectionsVersion", | ||
"org.scala-lang:scala-library:$scalaVersion.$scalaVersionRevision", | ||
"org.scala-lang:scala-reflect:$scalaVersion.$scalaVersionRevision" | ||
"org.scala-lang:scala-library:$scalaVersion", | ||
"org.scala-lang:scala-reflect:$scalaVersion" | ||
} | ||
} | ||
configurations.zinc { | ||
|
@@ -149,7 +149,7 @@ configure(allProjs) { | |
"-language:implicitConversions", "-language:existentials", "-language:postfixOps" | ||
] | ||
} | ||
compileScala.scalaCompileOptions.additionalParameters += "-optimize" | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why remove optimization option? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
compileScala.scalaCompileOptions.additionalParameters += ["-opt:l:inline", "-opt-inline-from:**"] | ||
[compileJava, compileTestJava]*.options.collect { options -> options.encoding = 'UTF-8' } | ||
|
||
jar { | ||
|
@@ -161,6 +161,7 @@ configure(allProjs) { | |
} | ||
|
||
scalaStyle { | ||
scalaVersion = '$scalaVersion' | ||
configLocation = "$rootProject.rootDir/gradle/scalastyle-config.xml" | ||
includeTestSourceDirectory = true | ||
source = "src/main/scala" | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -21,13 +21,14 @@ dependencies { | |
compile "org.apache.lucene:lucene-suggest:$luceneVersion" | ||
|
||
// Scopt | ||
compile "com.github.scopt:scopt_$scalaVersion:$scoptVersion" | ||
compile "com.github.scopt:scopt_%%:$scoptVersion" | ||
|
||
// Zip util | ||
compile 'org.zeroturnaround:zt-zip:1.14' | ||
|
||
// XGBoost | ||
compile ("ml.dmlc:xgboost4j-spark:$xgboostVersion") { exclude group: 'com.esotericsoftware.kryo', module: 'kryo' } | ||
compile ("ml.dmlc:xgboost4j_%%:$xgboostVersion") { exclude group: 'com.esotericsoftware.kryo', module: 'kryo' } | ||
compile ("ml.dmlc:xgboost4j-spark_%%:$xgboostVersion") { exclude group: 'com.esotericsoftware.kryo', module: 'kryo' } | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think we need both There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
W.r.t. GPUs: let's leave that baby alone for now and get this upgrade done first. 😅 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. When you finish this PR you can aim at an e2e on GPU throughout data transformations and Xgboost 😎 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. yes @gerashegalov i was trying to get people to use rapids ;) There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. On the Spark side we require 3.x , so you are on the right track @TuanNguyen27 |
||
// Akka slfj4 logging (version matches XGBoost dependency) | ||
testCompile "com.typesafe.akka:akka-slf4j_$scalaVersion:$akkaSlf4jVersion" | ||
testCompile "com.typesafe.akka:akka-slf4j_%%:$akkaSlf4jVersion" | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Initially I wanted to go with cross-compilation for 2.11/2.12, but now that we're upgrading to Spark 3 too we'll only build for 2.12.
This cross-compilation plugin seems to work well, though, so keeping it around for future 2.13 support.