Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .git-blame-ignore-revs
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# Scala Steward: Reformat with scalafmt 3.10.4
2ec2174c423f52a000a6d76f33d5236fba36a9e7
2 changes: 1 addition & 1 deletion .scalafmt.conf
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
version = "3.7.17"
version = "3.10.4"

align.preset = more
maxColumn = 100
Expand Down
28 changes: 15 additions & 13 deletions build.sc
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ implicit def millModuleBasePath: define.BasePath =
trait AmmSparkPublishModule extends PublishModule {
import mill.scalalib.publish._
def publishVersion = T {
val v = VcsVersion.vcsState().format()
val dirtyIdx = v.indexOf("-DIRTY")
val v = VcsVersion.vcsState().format()
val dirtyIdx = v.indexOf("-DIRTY")
def endsWithCommitHash =
v.length > 6 && v.substring(v.length - 6).forall(c => c.isDigit || (c >= 'a' && c <= 'f'))
if (dirtyIdx >= 0) v.take(dirtyIdx) + "-SNAPSHOT"
Expand Down Expand Up @@ -92,7 +92,7 @@ trait WithDependencyResourceFile extends JavaModule {
val dir = T.dest / "dep-file"
val dest = dir / dependencyResourcePath

val deps0 = T.task(compileIvyDeps() ++ transitiveIvyDeps())()
val deps0 = T.task(compileIvyDeps() ++ transitiveIvyDeps())()
val (_, res) = mill.modules.Jvm.resolveDependenciesMetadata(
repositoriesTask(),
deps0.map(resolveCoursierDependency().apply(_)),
Expand Down Expand Up @@ -125,14 +125,14 @@ trait WithDependencyResourceFile extends JavaModule {

object `spark-stubs_24` extends SbtModule with AmmSparkPublishModule {
def scalaVersion = Versions.scala212
def ivyDeps = super.ivyDeps() ++ Agg(
def ivyDeps = super.ivyDeps() ++ Agg(
Deps.sparkSql(scalaVersion())
)
}

object `spark-stubs_30` extends SbtModule with AmmSparkPublishModule {
def scalaVersion = Versions.scala212
def ivyDeps = super.ivyDeps() ++ Agg(
def ivyDeps = super.ivyDeps() ++ Agg(
Deps.sparkSql3
)
}
Expand All @@ -148,7 +148,7 @@ class SparkStubs32(val crossScalaVersion: String) extends CrossSbtModule
object core extends Cross[Core](Versions.scala: _*)
class Core(val crossScalaVersion: String) extends CrossSbtModule with WithPropertyFile
with AmmSparkPublishModule with AmmSparkMima {
def artifactName = "ammonite-spark"
def artifactName = "ammonite-spark"
def compileIvyDeps = super.compileIvyDeps() ++ Agg(
Deps.ammoniteReplApi,
Deps.sparkSql(scalaVersion())
Expand All @@ -158,16 +158,18 @@ class Core(val crossScalaVersion: String) extends CrossSbtModule with WithProper
Deps.jettyServer
)
def propertyFilePath =
os.sub / "org" / "apache" / "spark" / "sql" / "ammonitesparkinternals" / "ammonite-spark.properties"
os.sub / "org" / "apache" / "spark" / "sql" / "ammonitesparkinternals" /
"ammonite-spark.properties"
def versionInProperties = publishVersion()
def propResourcesDir = T.persistent {
def propResourcesDir = T.persistent {
import sys.process._

val dir = T.dest / "property-resources"
val ver = publishVersion()

val f =
dir / "org" / "apache" / "spark" / "sql" / "ammonitesparkinternals" / "ammonite-spark.properties"
dir / "org" / "apache" / "spark" / "sql" / "ammonitesparkinternals" /
"ammonite-spark.properties"

val contentStr =
s"""commit-hash=${Seq("git", "rev-parse", "HEAD").!!.trim}
Expand Down Expand Up @@ -225,7 +227,7 @@ class Tests(val crossScalaVersion: String) extends CrossSbtModule with WithPrope
object `local-spark-distrib-tests` extends SbtModule {
private def sv = Versions.scala212
def scalaVersion = sv
def moduleDeps = super.moduleDeps ++ Seq(
def moduleDeps = super.moduleDeps ++ Seq(
tests(sv)
)

Expand All @@ -235,14 +237,14 @@ object `local-spark-distrib-tests` extends SbtModule {
object `standalone-tests` extends SbtModule {
private def sv = Versions.scala212
def scalaVersion = sv
def moduleDeps = super.moduleDeps ++ Seq(
def moduleDeps = super.moduleDeps ++ Seq(
tests(sv)
)

object test extends Tests with AmmSparkTests
}

object `yarn-tests` extends Cross[YarnTests](Versions.scala: _*)
object `yarn-tests` extends Cross[YarnTests](Versions.scala: _*)
class YarnTests(val crossScalaVersion: String) extends CrossSbtModule {
def moduleDeps = super.moduleDeps ++ Seq(
tests()
Expand All @@ -254,7 +256,7 @@ class YarnTests(val crossScalaVersion: String) extends CrossSbtModule {
object `yarn-spark-distrib-tests` extends SbtModule {
private def sv = Versions.scala212
def scalaVersion = sv
def moduleDeps = super.moduleDeps ++ Seq(
def moduleDeps = super.moduleDeps ++ Seq(
tests(sv)
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ object DataFrameRenderer {
import scalatags.Text.all._
val columnFields = df.schema.fieldNames.toSeq.map(th(_))
val columns = tr(columnFields)
val rows = df
val rows = df
.rdd
.map { row =>
val fieldValues = row.toSeq.map(fieldToString).map(td(_))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,8 @@ final class ProgressSparkListener(
}
else
publish.stderr(
s"Stage $stageId not found (only have ${elems.asScala.toVector.map(_._1).sorted})" + '\n'
s"Stage $stageId not found (only have ${elems.asScala.toVector.map(_._1).sorted})" +
'\n'
)
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ final class SendLog(

res match {
case Some(t) => t
case None =>
case None =>
assert(delay.isFinite)
Thread.sleep(delay.toMillis)
helper((backOffFactor * delay).min(maxBackOffDelay))
Expand Down Expand Up @@ -111,7 +111,7 @@ final class SendLog(
while (keepReading && lines.length <= lineBufferSize && br.ready())
lines += br.readLine()

val l = lines.result()
val l = lines.result()
val l0 =
if (replaceHome) {
val home = sys.props("user.home")
Expand Down Expand Up @@ -193,7 +193,7 @@ object SendLog {

lazy val sendLog = new SendLog(f, commHandler, prefix = Option(prefix))

val id = Id.generate()
val id = Id.generate()
val data = DisplayData(
Map(DisplayData.ContentType.text -> "", DisplayData.ContentType.html -> ""),
idOpt = Some(id)
Expand All @@ -203,7 +203,7 @@ object SendLog {
commHandler.receiver(
commName,
onOpen = (_, _) => {
val msg = "See your browser developer console for detailed spark logs."
val msg = "See your browser developer console for detailed spark logs."
val updatedData = DisplayData(
Map(
DisplayData.ContentType.text -> msg,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,9 @@ final class StageElem(
| <div class="progress-bar" role="progressbar" style="background-color: blue; width: $donePct%; ${extraStyle.mkString(
"; "
)}; color: white" aria-valuenow="$donePct" aria-valuemin="0" aria-valuemax="100">
| $doneTasks0${if (diff == 0) "" else s" + $diff"} / $numTasks
| $doneTasks0${
if (diff == 0) "" else s" + $diff"
} / $numTasks
| </div>
| <div class="progress-bar" role="progressbar" style="background-color: red; width: $startedPct%" aria-valuenow="$startedPct" aria-valuemin="0" aria-valuemax="100"></div>
|</div>
Expand All @@ -97,7 +99,7 @@ final class StageElem(
)
}
else {
val taskOrTasks = if (numTasks <= 1) "task" else "tasks"
val taskOrTasks = if (numTasks <= 1) "task" else "tasks"
val onGoingMessage =
if (onGoingCount <= 0) ""
else s", $onGoingCount on-going"
Expand All @@ -109,7 +111,7 @@ final class StageElem(

if (stageDone0 && !keep) {
// Allow the user to see the completed bar before wiping it
val delay = 3.seconds
val delay = 3.seconds
val runnable: Runnable =
() =>
try {
Expand All @@ -130,11 +132,11 @@ final class StageElem(

object StageElem {
private def keepAlive = 30.seconds
lazy val scheduler = {
lazy val scheduler = {
val executor = new ScheduledThreadPoolExecutor(
1,
new ThreadFactory {
val count = new AtomicInteger
val count = new AtomicInteger
override def newThread(r: Runnable): Thread = {
val name = s"almond-spark-progress-${count.getAndIncrement()}"
val t = new Thread(r, name)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -404,7 +404,7 @@ class AmmoniteSparkSessionBuilder(implicit
.toSet

def toBeSent(jars: Seq[URI]): Seq[URI] = {
val nonSparkJars = jars.filter(uri => !sparkJarFileSet.contains(normalize(uri)))
val nonSparkJars = jars.filter(uri => !sparkJarFileSet.contains(normalize(uri)))
val nonSparkJars0 =
if (sendSourceJars0) nonSparkJars
else nonSparkJars.filter(uri => !uri.toASCIIString.endsWith("-sources.jar"))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ object SparkDependencies {
@tailrec
def addIfClasses(module: String, classNames: List[String]): Unit =
classNames match {
case Nil =>
case Nil =>
case h :: t =>
try {
cl.loadClass(h)
Expand All @@ -96,7 +96,7 @@ object SparkDependencies {
}

def stubsDependency = {
val sv = org.apache.spark.SPARK_VERSION
val sv = org.apache.spark.SPARK_VERSION
val suffix = sv.split('.').take(2) match {
case Array("2", n) if Try(n.toInt).toOption.exists(_ <= 3) =>
"20"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ class ExecutorClassLoader(
private val fetchFn: (String) => InputStream = uri.getScheme() match {
case "spark" => getClassFileInputStreamFromSparkRPC
case "http" | "https" | "ftp" => getClassFileInputStreamFromHttpServer
case _ =>
case _ =>
val fileSystem = FileSystem.get(uri, SparkHadoopUtil.get.newConfiguration(conf))
getClassFileInputStreamFromFileSystem(fileSystem)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ class ExecutorClassLoader(
private val fetchFn: (String) => InputStream = uri.getScheme() match {
case "spark" => getClassFileInputStreamFromSparkRPC
case "http" | "https" | "ftp" => getClassFileInputStreamFromHttpServer
case _ =>
case _ =>
val fileSystem = FileSystem.get(uri, SparkHadoopUtil.get.newConfiguration(conf))
getClassFileInputStreamFromFileSystem(fileSystem)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ class ExecutorClassLoader(
private val fetchFn: (String) => InputStream = uri.getScheme() match {
case "spark" => getClassFileInputStreamFromSparkRPC
case "http" | "https" | "ftp" => getClassFileInputStreamFromHttpServer
case _ =>
case _ =>
val fileSystem = FileSystem.get(uri, SparkHadoopUtil.get.newConfiguration(conf))
getClassFileInputStreamFromFileSystem(fileSystem)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ class ExecutorClassLoader(
private val fetchFn: (String) => InputStream = uri.getScheme() match {
case "spark" => getClassFileInputStreamFromSparkRPC
case "http" | "https" | "ftp" => getClassFileInputStreamFromHttpServer
case _ =>
case _ =>
val fileSystem = FileSystem.get(uri, SparkHadoopUtil.get.newConfiguration(conf))
getClassFileInputStreamFromFileSystem(fileSystem)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ class ProgressBarTests(sparkVersion: String, master: String, conf: (String, Stri

val tests = Tests {

"dummy test" - {
"dummy test" -
sparkSession(
if (interactive)
"""
Expand All @@ -47,7 +47,6 @@ class ProgressBarTests(sparkVersion: String, master: String, conf: (String, Stri
@ val rdd = spark.sparkContext.makeRDD(1 to 10000, 200)
"""
)
}

}

Expand Down
Loading
Loading