New Persistence API with higher scalability limits (d2iq-archive#4178)

Major changes

- New Persistence API with support for ZK, Memory, and Two caching modes.
- Move all storage related "stuff" into marathon/core/storage
- Default the storage backend to the new ZK API.
- Lazy storage of Groups (new API only) where Apps are stored separately.
- Only GroupRepository is allowed write access to AppRepository
- Completely unify the persistence API between new and old, keeping a single, simplified chain of Repositories.
- Make versioned repositories explicit.
- Prepare for a future where typesafe config is used as opposed to a million command line options (storage supports it now).
- Refactor migration a bit to differentiate new API vs old API (needs a bit more work)
- Run repository tests against all of the backing stores.
  (in some cases) remove mocks and prefer the in memory implementation where it otherwise makes sense (not to simulate errors, etc).
- Add ZookeeperServerTest that will spin up/down a zk server and provide a client on a ephemeral port. This has the major advantage of bringing us closer to being able to run tests in parallel.
- Add a Garbage Collector to the new persistence store that will try to enforce max versions, but in the background. The garbage collector is triggered by deployment plan deletion.

Minor changes

- run scalastyle for test in compile, not test.
- add plugin to auto-resolve documentation links
- remove lots of Guice Injections for Storage.
- Use immutable.Seq across the persistence layer.
- Use OffsetDateTime (java8 time) for versioning in the new API as it is strictly better than timestamp.
- Make some traits have companion objects so they can be imported instead of mixed-in.
- Add a generic Sink stage that can drain into any collection, then wrap the Akka Sink API in a companion.
- Add some tooling for enriched Locks.
- Add scala friendly API around typesafe config.
- Add RichFuture that adds "asTry" which makes a future always complete, but with a try instead (very nice for async/await)
- Fix some oversight issues with CapConcurrentExecutions. I would like to remove the actor from this and make it a functional transform instead.
- Add a SettableClock (the java8 time Clock) that lets you control time a bit easier for testing.
- default sbt opts to use better jvm options
- run-tests.sh runs all of the SBT commands together.

* Move PersistentStoreTests to use built-in zk

.sbtopts

@@ -1,5 +1,15 @@
--mem 4096
+-mem 8192
 -J-XX:MaxMetaspaceSize=512m
 -J-XX:+UseConcMarkSweepGC
 -J-XX:+CMSClassUnloadingEnabled
-
+-J-server
+-J-XX:+CMSParallelRemarkEnabled
+-J-XX:+UseCMSInitiatingOccupancyOnly
+-J-XX:+ScavengeBeforeFullGC
+-J-XX:+CMSScavengeBeforeRemark
+-J-XX:+ScavengeBeforeFullGC
+-J-XX:+CMSScavengeBeforeRemark
+-J-XX:SoftRefLRUPolicyMSPerMB=50
+-J-XX:CMSInitiatingOccupancyFraction=70
+-J-XX:-OmitStackTraceInFastThrow
+-J-XX:+AggressiveOpts

.travis.yml

@@ -3,16 +3,16 @@ language: scala
 jdk:
   - oraclejdk8
 scala:
-  - 2.11.7
+  - 2.11.8
 cache:
   directories:
     - $HOME/.sbt
     - $HOME/.ivy2
 script:
-  - sbt clean coverage doc assembly
+  - sbt "set parallelExecution in Test := false" clean coverage doc assembly
 before_script: # the automated download fails sometimes
-    - mkdir -p $HOME/.sbt/launchers/0.13.8/
-    - test -r $HOME/.sbt/launchers/0.13.8/sbt-launch.jar || curl -L -o $HOME/.sbt/launchers/0.13.8/sbt-launch.jar http://dl.bintray.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/0.13.8/sbt-launch.jar
+    - mkdir -p $HOME/.sbt/launchers/0.13.11/
+    - test -r $HOME/.sbt/launchers/0.13.11/sbt-launch.jar || curl -L -o $HOME/.sbt/launchers/0.13.11/sbt-launch.jar http://dl.bintray.com/typesafe/ivy-releases/org.scala-sbt/sbt-launch/0.13.11/sbt-launch.jar
 after_success:
   - sbt coverageReport coveralls
 notifications:

bin/run-tests.sh

@@ -159,8 +159,7 @@ fi
 
 DOCKER_IMAGE="marathon-buildbase:$BUILD_ID"
 
-DOCKER_CMD='sbt -Dsbt.log.format=false test integration:test &&
-  sbt -Dsbt.log.format=false "project mesos-simulation" integration:test "test:runMain mesosphere.mesos.scale.DisplayAppScalingResults"'
+DOCKER_CMD='sbt -Dsbt.log.format=false "; test; integration:test; mesos-simulation/test; mesos-simulation/integration:test; mesos-simulation/test:runMain mesosphere.mesos.scale.DisplayAppScalingResults"'
 
 DOCKER_ARGS=( ${DOCKER_OPTIONS[@]} "$DOCKER_IMAGE" -c "$DOCKER_CMD" )
 

project/build.scala

@@ -53,8 +53,7 @@ object MarathonBuild extends Build {
             val suffix = Try(Process("git diff --shortstat").lines.headOption.map(_ => "-dev")).toOption.flatten.getOrElse("")
             Try(Process("git rev-parse HEAD").lines.headOption).toOption.flatten.getOrElse("unknown") + suffix
           }),
-        buildInfoPackage := "mesosphere.marathon",
-        fork in Test := true
+        buildInfoPackage := "mesosphere.marathon"
       )
   )
     .configs(IntegrationTest, Benchmark)
@@ -76,17 +75,6 @@ object MarathonBuild extends Build {
 
   /**
    * Determine scala test runner output. `-e` for reporting on standard error.
-   *
-   * W - without color
-   * D - show all durations
-   * S - show short stack traces
-   * F - show full stack traces
-   * U - unformatted mode
-   * I - show reminder of failed and canceled tests without stack traces
-   * T - show reminder of failed and canceled tests with short stack traces
-   * G - show reminder of failed and canceled tests with full stack traces
-   * K - exclude TestCanceled events from reminder
-   *
    * http://scalatest.org/user_guide/using_the_runner
    */
   lazy val formattingTestArg = Tests.Argument("-eDFG")
@@ -104,6 +92,7 @@ object MarathonBuild extends Build {
   // where they can kill each other _even_ when in different processes.
   lazy val integrationTestSettings = inConfig(IntegrationTest)(Defaults.testTasks) ++
     Seq(
+      fork in IntegrationTest := true,
       testOptions in IntegrationTest := Seq(formattingTestArg, Tests.Argument("-n", "mesosphere.marathon.IntegrationTest")),
       parallelExecution in IntegrationTest := false,
       testForkedParallel in IntegrationTest := false,
@@ -119,7 +108,8 @@ object MarathonBuild extends Build {
     )
 
   lazy val testSettings = Seq(
-    parallelExecution in Test := false,
+    parallelExecution in Test := true,
+    testForkedParallel in Test := true,
     testOptions in Test := Seq(formattingTestArg, Tests.Argument("-l", "mesosphere.marathon.IntegrationTest")),
     fork in Test := true
   )
@@ -130,7 +120,7 @@ object MarathonBuild extends Build {
     testScalaStyle := {
       org.scalastyle.sbt.ScalastylePlugin.scalastyle.in(Compile).toTask("").value
     },
-    (test in Test) <<= (test in Test) dependsOn testScalaStyle
+    (compile in Test) <<= (compile in Test) dependsOn testScalaStyle
   )
 
   lazy val IntegrationTest = config("integration") extend Test
@@ -160,8 +150,7 @@ object MarathonBuild extends Build {
       "Spray Maven Repository"    at "http://repo.spray.io/"
     ),
     cancelable in Global := true,
-    fork in Test := true,
-    javaOptions += "-Xmx4G"
+    javaOptions += "-Xmx8G"
   )
 
   lazy val asmSettings = Seq(
@@ -280,6 +269,7 @@ object Dependencies {
     akkaActor % "compile",
     akkaSlf4j % "compile",
     akkaStream % "compile",
+    akkaHttp % "compile",
     asyncAwait % "compile",
     sprayClient % "compile",
     sprayHttpx % "compile",
@@ -309,6 +299,7 @@ object Dependencies {
     curatorClient % "compile",
     curatorFramework % "compile",
     java8Compat % "compile",
+    scalaLogging % "compile",
     logstash % "compile",
 
     // test
@@ -352,7 +343,8 @@ object Dependency {
     val Logstash = "4.7"
     val WixAccord = "0.5"
     val Curator = "2.11.0"
-    val Java8Compat = "0.8.0-RC1"
+    val Java8Compat = "0.8.0-RC3"
+    val ScalaLogging = "3.4.0"
 
     // test deps versions
     val Mockito = "1.10.19"
@@ -367,6 +359,7 @@ object Dependency {
   val akkaActor = "com.typesafe.akka" %% "akka-actor" % V.Akka
   val akkaSlf4j = "com.typesafe.akka" %% "akka-slf4j" % V.Akka
   val akkaStream = "com.typesafe.akka" %% "akka-stream" % V.Akka
+  val akkaHttp = "com.typesafe.akka" %% "akka-http-experimental" % V.Akka
   val asyncAwait = "org.scala-lang.modules" %% "scala-async" % V.AsyncAwait
   val sprayClient = "io.spray" %% "spray-client" % V.Spray
   val sprayHttpx = "io.spray" %% "spray-httpx" % V.Spray
@@ -399,6 +392,7 @@ object Dependency {
   val curatorClient = "org.apache.curator" % "curator-client" % V.Curator
   val curatorFramework = "org.apache.curator" % "curator-framework" % V.Curator
   val java8Compat = "org.scala-lang.modules" %% "scala-java8-compat" % V.Java8Compat
+  val scalaLogging = "com.typesafe.scala-logging" %% "scala-logging" % V.ScalaLogging
 
   object Test {
     val scalatest = "org.scalatest" %% "scalatest" % V.ScalaTest

project/plugins.sbt

@@ -30,3 +30,4 @@ addSbtPlugin("org.scoverage" % "sbt-coveralls" % "1.1.0")
 
 addSbtPlugin("com.timushev.sbt" % "sbt-updates" % "0.1.10")
 
+addSbtPlugin("com.thoughtworks.sbt-api-mappings" % "sbt-api-mappings" % "latest.release")

scalastyle-config.xml

@@ -115,11 +115,6 @@
  </check>
  <check level="error" class="org.scalastyle.scalariform.DeprecatedJavaChecker" enabled="true"/>
  <check level="error" class="org.scalastyle.scalariform.EmptyClassChecker" enabled="true"/>
- <check level="error" class="org.scalastyle.scalariform.ClassTypeParameterChecker" enabled="true">
-  <parameters>
-   <parameter name="regex"><![CDATA[^[A-Z_]$]]></parameter>
-  </parameters>
- </check>
  <check level="error" class="org.scalastyle.scalariform.UnderscoreImportChecker" enabled="false"/>
  <check level="error" class="org.scalastyle.scalariform.LowercasePatternMatchChecker" enabled="true"/>
  <check level="error" class="org.scalastyle.scalariform.MultipleStringLiteralsChecker" enabled="false">