--- 1000/config.json
+++ 1100/config.json
@@ -69,7 +69,7 @@
},
"docker-image": {
"type": "string",
- "description": "The docker image used to run the dispatcher, drivers, and executors. If no image is specified mesosphere/spark:2.11.0-2.4.6-scala-2.11-hadoop-2.9 will be used. If, for example, you need a Spark built with a specific Hadoop version, set this variable to one of the images here: https://hub.docker.com/r/mesosphere/spark/tags/",
+ "description": "The docker image used to run the dispatcher, drivers, and executors. If no image is specified mesosphere/spark:2.12.0-3.0.1-scala-2.12-hadoop-3.2 will be used. If, for example, you need a Spark built with a specific Hadoop version, set this variable to one of the images here: https://hub.docker.com/r/mesosphere/spark/tags/",
"default": ""
},
"log-level": {
@@ -86,6 +86,22 @@
"TRACE",
"ALL"
]
+ },
+ "logrotate_options": {
+ "description": "The logrotate options for the DC/OS service.",
+ "type": "object",
+ "properties": {
+ "stdout_max_size": {
+ "description": "Logrotate stdout max size. Sizes must be an integer of less than 2^64 and must be suffixed with a unit such as B (bytes), KB, MB, GB, or TB. There should be no whitespace between the integer and unit.",
+ "type": "string",
+ "default": "8MB"
+ },
+ "stderr_max_size": {
+ "description": "Logrotate stderr max size. Sizes must be an integer of less than 2^64 and must be suffixed with a unit such as B (bytes), KB, MB, GB, or TB. There should be no whitespace between the integer and unit.",
+ "type": "string",
+ "default": "8MB"
+ }
+ }
},
"virtual_network_enabled": {
"description": "Enable virtual networking",
--- 1000/marathon.json.mustache
+++ 1100/marathon.json.mustache
@@ -51,7 +51,9 @@
"SPARK_DISPATCHER_MESOS_ROLE": "{{service.role}}",
"SPARK_DISPATCHER_ENFORCE_ROLE": "{{service.enforce_role}}",
"SPARK_DISPATCHER_MESOS_PRINCIPAL": "{{service.service_account}}",
- "SPARK_DISPATCHER_MESOS_SECRET": "{{service.secret}}",
+ "SPARK_DISPATCHER_MESOS_SECRET": "{{service.service_account_secret}}",
+ "CONTAINER_LOGGER_LOGROTATE_MAX_STDOUT_SIZE": "{{service.logrotate_options.stdout_max_size}}",
+ "CONTAINER_LOGGER_LOGROTATE_MAX_STDERR_SIZE": "{{service.logrotate_options.stderr_max_size}}",
"SPARK_LOG_LEVEL": "{{service.log-level}}"
},
--- 1000/package.json
+++ 1100/package.json
@@ -1,17 +1,17 @@
{
"packagingVersion": "4.0",
"name": "spark",
- "version": "2.11.0-2.4.6",
+ "version": "2.12.0-3.0.1",
"scm": "https://github.com/mesosphere/spark-build.git",
"maintainer": "support@mesosphere.io",
- "website": "https://docs.mesosphere.com/services/spark/2.11.0-2.4.6/",
- "description": "Apache Spark is a unified analytics engine for large-scale data processing. Documentation: https://docs.mesosphere.com/services/spark/2.11.0-2.4.6/",
+ "website": "https://docs.mesosphere.com/services/spark/2.12.0-3.0.1/",
+ "description": "Apache Spark is a unified analytics engine for large-scale data processing. Documentation: https://docs.mesosphere.com/services/spark/2.12.0-3.0.1/",
"framework": true,
"upgradesFrom": [],
"downgradesTo": [],
"preInstallNotes": "Default configuration requires 1 agent node with: 1 CPU | 1024 MB MEM",
- "postInstallNotes": "DC/OS Spark is being installed!\n\n\tDocumentation: https://docs.mesosphere.com/services/spark/2.11.0-2.4.6/\n\tIssues: https://docs.mesosphere.com/support/",
- "postUninstallNotes": "The Apache Spark DC/OS Service has been uninstalled and will no longer run.\nPlease follow the instructions at https://docs.mesosphere.com/services/spark/2.11.0-2.4.6/uninstall/ to clean up any persisted state.",
+ "postInstallNotes": "DC/OS Spark is being installed!\n\n\tDocumentation: https://docs.mesosphere.com/services/spark/2.12.0-3.0.1/\n\tIssues: https://docs.mesosphere.com/support/",
+ "postUninstallNotes": "The Apache Spark DC/OS Service has been uninstalled and will no longer run.\nPlease follow the instructions at https://docs.mesosphere.com/services/spark/2.12.0-3.0.1/uninstall/ to clean up any persisted state.",
"tags": [
"bigdata",
"mapreduce",
--- 1000/resource.json
+++ 1100/resource.json
@@ -2,7 +2,7 @@
"assets": {
"container": {
"docker": {
- "spark_docker": "mesosphere/spark:2.11.0-2.4.6-scala-2.11-hadoop-2.9"
+ "spark_docker": "mesosphere/spark:2.12.0-3.0.1-scala-2.12-hadoop-3.2"
}
}
},
@@ -18,11 +18,11 @@
"contentHash": [
{
"algo": "sha256",
- "value": "0502eb1ff7d74262f021f83677752213e11c03a739b2dab288b3370f4c1f4b9e"
+ "value": "a724ebf0d1c1dfc31b7f630a087d575f59d3672b507361703f0da1335cc61bfd"
}
],
"kind": "executable",
- "url": "https://downloads.mesosphere.com/spark/assets/2.11.0-2.4.6/dcos-spark-darwin"
+ "url": "https://downloads.mesosphere.com/spark/assets/2.12.0-3.0.1/dcos-spark-darwin"
}
},
"linux": {
@@ -30,11 +30,11 @@
"contentHash": [
{
"algo": "sha256",
- "value": "cb59b6c090476f07a3198d0a7e7e926594509de5b45630aa81414a83295c0c71"
+ "value": "3ef41ceda4a024274dd48f3d6a271771c0bb2015e0a80f738c4cf2386b128635"
}
],
"kind": "executable",
- "url": "https://downloads.mesosphere.com/spark/assets/2.11.0-2.4.6/dcos-spark-linux"
+ "url": "https://downloads.mesosphere.com/spark/assets/2.12.0-3.0.1/dcos-spark-linux"
}
},
"windows": {
@@ -42,11 +42,11 @@
"contentHash": [
{
"algo": "sha256",
- "value": "2f0fe51bf3cca285c034763822556c04c30003ccdf91c787b5192e0e03ac440b"
+ "value": "ec354bbbb9a8b3ab81e3b6e4691eb9a6fe3d7bb6dbc33ef583e73d30c9016f79"
}
],
"kind": "executable",
- "url": "https://downloads.mesosphere.com/spark/assets/2.11.0-2.4.6/dcos-spark.exe"
+ "url": "https://downloads.mesosphere.com/spark/assets/2.12.0-3.0.1/dcos-spark.exe"
}
}
}
Release spark 2.12.0-3.0.1 (automated commit)
Description: Source URL: https://infinity-artifacts.s3.amazonaws.com/permanent/spark/2.12.0-3.0.1/stub-universe-spark.json
Changes between revisions 1000 => 1100: 0 files added: [] 0 files removed: [] 4 files changed: