diff --git a/connector/kafka-0-10-sql/pom.xml b/connector/kafka-0-10-sql/pom.xml index a1a85ac985f9..dd4e6f1e7e99 100644 --- a/connector/kafka-0-10-sql/pom.xml +++ b/connector/kafka-0-10-sql/pom.xml @@ -132,8 +132,8 @@ test - org.eclipse.jetty - jetty-servlet + org.eclipse.jetty.ee10 + jetty-ee10-servlet ${jetty.version} test diff --git a/core/pom.xml b/core/pom.xml index 69dde0d9cc05..1ac5081e9e26 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -138,8 +138,8 @@ - org.eclipse.jetty - jetty-plus + org.eclipse.jetty.ee10 + jetty-ee10-plus compile @@ -163,13 +163,13 @@ compile - org.eclipse.jetty - jetty-servlet + org.eclipse.jetty.ee10 + jetty-ee10-servlet compile - org.eclipse.jetty - jetty-proxy + org.eclipse.jetty.ee10 + jetty-ee10-proxy compile @@ -178,8 +178,8 @@ compile - org.eclipse.jetty - jetty-servlets + org.eclipse.jetty.ee10 + jetty-ee10-servlets compile @@ -541,7 +541,7 @@ true true - guava,protobuf-java,jetty-io,jetty-servlet,jetty-servlets,jetty-http,jetty-plus,jetty-util,jetty-server,jetty-security,jetty-proxy,jetty-client + guava,protobuf-java,jetty-io,jetty-ee10-servlet,jetty-ee10-servlets,jetty-http,jetty-ee10-plus,jetty-util,jetty-server,jetty-security,jetty-proxy,jetty-client true diff --git a/core/src/main/scala/org/apache/spark/TestUtils.scala b/core/src/main/scala/org/apache/spark/TestUtils.scala index aadfb2125cd6..1bfc6c3d0fad 100644 --- a/core/src/main/scala/org/apache/spark/TestUtils.scala +++ b/core/src/main/scala/org/apache/spark/TestUtils.scala @@ -41,11 +41,8 @@ import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.core.LoggerContext import org.apache.logging.log4j.core.appender.ConsoleAppender import org.apache.logging.log4j.core.config.builder.api.ConfigurationBuilderFactory -import org.eclipse.jetty.server.Handler import org.eclipse.jetty.server.Server -import org.eclipse.jetty.server.handler.DefaultHandler -import org.eclipse.jetty.server.handler.HandlerList -import org.eclipse.jetty.server.handler.ResourceHandler +import org.eclipse.jetty.server.handler.{ContextHandler, ContextHandlerCollection, DefaultHandler, ResourceHandler} import org.json4s.JsonAST.JValue import org.json4s.jackson.JsonMethods.{compact, render} @@ -333,9 +330,12 @@ private[spark] object TestUtils extends SparkTestUtils { // 0 as port means choosing randomly from the available ports val server = new Server(new InetSocketAddress(Utils.localCanonicalHostName(), 0)) val resHandler = new ResourceHandler() - resHandler.setResourceBase(resBaseDir) - val handlers = new HandlerList() - handlers.setHandlers(Array[Handler](resHandler, new DefaultHandler())) + resHandler.setBaseResourceAsString(resBaseDir) + val resCxt = new ContextHandler + resCxt.setContextPath("/") + resCxt.setHandler(resHandler) + val handlers = new ContextHandlerCollection() + handlers.setHandlers(resCxt, new DefaultHandler()) server.setHandler(handlers) server.start() try { diff --git a/core/src/main/scala/org/apache/spark/deploy/history/ApplicationCache.scala b/core/src/main/scala/org/apache/spark/deploy/history/ApplicationCache.scala index e979671890ea..c2fe0acec2e9 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/ApplicationCache.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/ApplicationCache.scala @@ -26,7 +26,7 @@ import com.google.common.cache.{CacheBuilder, CacheLoader, LoadingCache, Removal import com.google.common.util.concurrent.UncheckedExecutionException import jakarta.servlet.{DispatcherType, Filter, FilterChain, ServletException, ServletRequest, ServletResponse} import jakarta.servlet.http.{HttpServletRequest, HttpServletResponse} -import org.eclipse.jetty.servlet.FilterHolder +import org.eclipse.jetty.ee10.servlet.FilterHolder import org.apache.spark.internal.Logging import org.apache.spark.internal.LogKeys._ diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala index 14daa5d88b1c..a4e047f7683a 100644 --- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala +++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala @@ -23,7 +23,7 @@ import scala.util.control.NonFatal import scala.xml.Node import jakarta.servlet.http.{HttpServlet, HttpServletRequest, HttpServletResponse} -import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder} +import org.eclipse.jetty.ee10.servlet.{ServletContextHandler, ServletHolder} import org.apache.spark.{SecurityManager, SparkConf} import org.apache.spark.deploy.SparkHadoopUtil diff --git a/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionServer.scala b/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionServer.scala index e172a06f0a32..ee404ac11a96 100644 --- a/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionServer.scala +++ b/core/src/main/scala/org/apache/spark/deploy/rest/RestSubmissionServer.scala @@ -25,8 +25,8 @@ import scala.io.Source import com.fasterxml.jackson.core.JsonProcessingException import jakarta.servlet.DispatcherType import jakarta.servlet.http.{HttpServlet, HttpServletRequest, HttpServletResponse} +import org.eclipse.jetty.ee10.servlet.{FilterHolder, ServletContextHandler, ServletHolder} import org.eclipse.jetty.server.{HttpConfiguration, HttpConnectionFactory, Server, ServerConnector} -import org.eclipse.jetty.servlet.{FilterHolder, ServletContextHandler, ServletHolder} import org.eclipse.jetty.util.thread.{QueuedThreadPool, ScheduledExecutorScheduler} import org.json4s._ import org.json4s.jackson.JsonMethods._ diff --git a/core/src/main/scala/org/apache/spark/metrics/MetricsSystem.scala b/core/src/main/scala/org/apache/spark/metrics/MetricsSystem.scala index af1f54007248..25736475be31 100644 --- a/core/src/main/scala/org/apache/spark/metrics/MetricsSystem.scala +++ b/core/src/main/scala/org/apache/spark/metrics/MetricsSystem.scala @@ -23,7 +23,7 @@ import java.util.concurrent.TimeUnit import scala.collection.mutable import com.codahale.metrics.{Metric, MetricRegistry} -import org.eclipse.jetty.servlet.ServletContextHandler +import org.eclipse.jetty.ee10.servlet.ServletContextHandler import org.apache.spark.{SecurityManager, SparkConf} import org.apache.spark.internal.Logging diff --git a/core/src/main/scala/org/apache/spark/metrics/sink/MetricsServlet.scala b/core/src/main/scala/org/apache/spark/metrics/sink/MetricsServlet.scala index 7d676cd2c27b..305b391cc5fc 100644 --- a/core/src/main/scala/org/apache/spark/metrics/sink/MetricsServlet.scala +++ b/core/src/main/scala/org/apache/spark/metrics/sink/MetricsServlet.scala @@ -24,7 +24,7 @@ import com.codahale.metrics.MetricRegistry import com.codahale.metrics.json.MetricsModule import com.fasterxml.jackson.databind.ObjectMapper import jakarta.servlet.http.HttpServletRequest -import org.eclipse.jetty.servlet.ServletContextHandler +import org.eclipse.jetty.ee10.servlet.ServletContextHandler import org.apache.spark.SparkConf import org.apache.spark.ui.JettyUtils._ diff --git a/core/src/main/scala/org/apache/spark/metrics/sink/PrometheusServlet.scala b/core/src/main/scala/org/apache/spark/metrics/sink/PrometheusServlet.scala index 2ab49eae8cd8..8252f235e582 100644 --- a/core/src/main/scala/org/apache/spark/metrics/sink/PrometheusServlet.scala +++ b/core/src/main/scala/org/apache/spark/metrics/sink/PrometheusServlet.scala @@ -21,7 +21,7 @@ import java.util.Properties import com.codahale.metrics.MetricRegistry import jakarta.servlet.http.HttpServletRequest -import org.eclipse.jetty.servlet.ServletContextHandler +import org.eclipse.jetty.ee10.servlet.ServletContextHandler import org.apache.spark.SparkConf import org.apache.spark.annotation.{DeveloperApi, Since, Unstable} diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala index 61e800844db0..c8717c97140d 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/ApiRootResource.scala @@ -22,8 +22,8 @@ import jakarta.servlet.ServletContext import jakarta.servlet.http.HttpServletRequest import jakarta.ws.rs._ import jakarta.ws.rs.core.{Context, Response} +import org.eclipse.jetty.ee10.servlet.{ServletContextHandler, ServletHolder} import org.eclipse.jetty.server.handler.ContextHandler -import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder} import org.glassfish.jersey.server.ServerProperties import org.glassfish.jersey.servlet.ServletContainer diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala index 6efe3106ba56..ea30d59832b6 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/PrometheusResource.scala @@ -18,7 +18,7 @@ package org.apache.spark.status.api.v1 import jakarta.ws.rs._ import jakarta.ws.rs.core.MediaType -import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder} +import org.eclipse.jetty.ee10.servlet.{ServletContextHandler, ServletHolder} import org.glassfish.jersey.server.ServerProperties import org.glassfish.jersey.servlet.ServletContainer diff --git a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala index bd7848c76437..a59c6ff9bd6c 100644 --- a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala @@ -18,22 +18,25 @@ package org.apache.spark.ui import java.net.{URI, URL, URLDecoder} -import java.util.EnumSet +import java.util.{EnumSet, List => JList} +import scala.jdk.CollectionConverters._ import scala.language.implicitConversions import scala.util.Try import scala.xml.Node -import jakarta.servlet.DispatcherType +import jakarta.servlet.{DispatcherType, Filter, FilterChain, ServletRequest, ServletResponse} import jakarta.servlet.http._ +import org.eclipse.jetty.client.{Response => CResponse} import org.eclipse.jetty.client.HttpClient -import org.eclipse.jetty.client.api.Response -import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP -import org.eclipse.jetty.proxy.ProxyServlet +import org.eclipse.jetty.client.transport.HttpClientTransportOverHTTP +import org.eclipse.jetty.ee10.proxy.ProxyServlet +import org.eclipse.jetty.ee10.servlet._ +import org.eclipse.jetty.http.{HttpField, HttpFields, HttpHeader} import org.eclipse.jetty.server._ -import org.eclipse.jetty.server.handler._ +import org.eclipse.jetty.server.handler.{ContextHandler, ContextHandlerCollection, ErrorHandler} import org.eclipse.jetty.server.handler.gzip.GzipHandler -import org.eclipse.jetty.servlet._ +import org.eclipse.jetty.util.{Callback, URIUtil} import org.eclipse.jetty.util.component.LifeCycle import org.eclipse.jetty.util.thread.{QueuedThreadPool, ScheduledExecutorScheduler} import org.json4s.JValue @@ -172,7 +175,7 @@ private[spark] object JettyUtils extends Logging { val holder = new ServletHolder(staticHandler) Option(Utils.getSparkClassLoader.getResource(resourceBase)) match { case Some(res) => - holder.setInitParameter("resourceBase", res.toString) + holder.setInitParameter("baseResource", res.toString) case None => throw new Exception("Could not find resource path for Web UI: " + resourceBase) } @@ -215,7 +218,7 @@ private[spark] object JettyUtils extends Logging { override def filterServerResponseHeader( clientRequest: HttpServletRequest, - serverResponse: Response, + serverResponse: CResponse, headerName: String, headerValue: String): String = { if (headerName.equalsIgnoreCase("location")) { @@ -266,8 +269,7 @@ private[spark] object JettyUtils extends Logging { val errorHandler = new ErrorHandler() errorHandler.setShowStacks(true) - errorHandler.setServer(server) - server.addBean(errorHandler) + server.setErrorHandler(errorHandler) val collection = new ContextHandlerCollection conf.get(PROXY_REDIRECT_URI) match { @@ -392,24 +394,11 @@ private[spark] object JettyUtils extends Logging { } private def createRedirectHttpsHandler(securePort: Int, scheme: String): ContextHandler = { - val redirectHandler: ContextHandler = new ContextHandler + val redirectHandler = new ServletContextHandler redirectHandler.setContextPath("/") redirectHandler.setVirtualHosts(toVirtualHosts(REDIRECT_CONNECTOR_NAME)) - redirectHandler.setHandler(new AbstractHandler { - override def handle( - target: String, - baseRequest: Request, - request: HttpServletRequest, - response: HttpServletResponse): Unit = { - if (baseRequest.isSecure) { - return - } - val httpsURI = createRedirectURI(scheme, securePort, baseRequest) - response.setContentLength(0) - response.sendRedirect(response.encodeRedirectURL(httpsURI)) - baseRequest.setHandled(true) - } - }) + redirectHandler.addFilter( + new FilterHolder(new HttpsRedirectFilter(scheme, securePort)), "/*", null) redirectHandler } @@ -441,14 +430,18 @@ private[spark] object JettyUtils extends Logging { headerValue: String, clientRequest: HttpServletRequest, targetUri: URI): String = { - val toReplace = targetUri.getScheme() + "://" + targetUri.getAuthority() - if (headerValue.startsWith(toReplace)) { - val id = clientRequest.getPathInfo.substring("/proxy/".length).takeWhile(_ != '/') - val headerPath = headerValue.substring(toReplace.length) - - s"${clientRequest.getScheme}://${clientRequest.getHeader("host")}/proxy/$id$headerPath" + val id = clientRequest.getPathInfo.substring(1).takeWhile(_ != '/') + val urlToId = s"${clientRequest.getScheme}://${clientRequest.getHeader("host")}/proxy/$id" + if (new URI(headerValue).getScheme != null) { + val toReplace = targetUri.getScheme() + "://" + targetUri.getAuthority() + if (headerValue.startsWith(toReplace)) { + val headerPath = headerValue.substring(toReplace.length) + urlToId + headerPath + } else { + null + } } else { - null + urlToId + headerValue } } @@ -462,37 +455,8 @@ private[spark] object JettyUtils extends Logging { handler.addFilter(holder, "/*", EnumSet.allOf(classOf[DispatcherType])) } - private def decodeURL(url: String, encoding: String): String = { - if (url == null) { - null - } else { - URLDecoder.decode(url, encoding) - } - } - - // Create a new URI from the arguments, handling IPv6 host encoding and default ports. - private def createRedirectURI(scheme: String, port: Int, request: Request): String = { - val server = request.getServerName - val redirectServer = if (server.contains(":") && !server.startsWith("[")) { - s"[${server}]" - } else { - server - } - val authority = s"$redirectServer:$port" - val queryEncoding = if (request.getQueryEncoding != null) { - request.getQueryEncoding - } else { - // By default decoding the URI as "UTF-8" should be enough for SparkUI - "UTF-8" - } - // The request URL can be raw or encoded here. To avoid the request URL being - // encoded twice, let's decode it here. - val requestURI = decodeURL(request.getRequestURI, queryEncoding) - val queryString = decodeURL(request.getQueryString, queryEncoding) - new URI(scheme, authority, requestURI, queryString, null).toString - } - - def toVirtualHosts(connectors: String*): Array[String] = connectors.map("@" + _).toArray + def toVirtualHosts(connectors: String*): JList[String] = + JList.of(connectors.map("@" + _).toArray: _*) } @@ -522,7 +486,7 @@ private[spark] case class ServerInfo( def removeHandler(handler: ServletContextHandler): Unit = synchronized { // Since addHandler() always adds a wrapping gzip handler, find the container handler // and remove it. - rootHandler.getHandlers() + rootHandler.getHandlers.asScala .find { h => h.isInstanceOf[GzipHandler] && h.asInstanceOf[GzipHandler].getHandler() == handler } @@ -597,36 +561,85 @@ private[spark] case class ServerInfo( * a servlet context without the trailing slash (e.g. "/jobs") - Jetty will send a redirect to the * same URL, but with a trailing slash. */ -private class ProxyRedirectHandler(_proxyUri: String) extends HandlerWrapper { +private class ProxyRedirectHandler(_proxyUri: String) extends Handler.Wrapper { private val proxyUri = _proxyUri.stripSuffix("/") - override def handle( - target: String, - baseRequest: Request, - request: HttpServletRequest, - response: HttpServletResponse): Unit = { - super.handle(target, baseRequest, request, new ResponseWrapper(request, response)) + override def handle(request: Request, response: Response, callback: Callback): Boolean = { + super.handle(request, new ResponseWrapper(request, response), callback) } - private class ResponseWrapper( - req: HttpServletRequest, - res: HttpServletResponse) - extends HttpServletResponseWrapper(res) { + private class ResponseWrapper(req: Request, res: Response) extends Response.Wrapper(req, res) { + override def getHeaders(): HttpFields.Mutable = { + new MutableFieldsWrapper(req, super.getHeaders()) + } + } - override def sendRedirect(location: String): Unit = { - val newTarget = if (location != null) { + private class MutableFieldsWrapper( + req: Request, + wrapped: HttpFields.Mutable) extends HttpFields.Mutable.Wrapper(wrapped) { + override def onAddField(field: HttpField): HttpField = { + if (field.is(HttpHeader.LOCATION.asString())) { + val location = field.getValue + // null check should be done by Jetty before adding Location header. + assert(location != null) val target = new URI(location) // The target path should already be encoded, so don't re-encode it, just the // proxy address part. val proxyBase = UIUtils.uiRoot(req) val proxyPrefix = if (proxyBase.nonEmpty) s"$proxyUri$proxyBase" else proxyUri - s"${res.encodeURL(proxyPrefix)}${target.getPath()}" + val newTarget = URIUtil.encodePath(proxyPrefix) + target.getPath() + new HttpField(field.getName(), newTarget) } else { - null + field } - super.sendRedirect(newTarget) } } +} + +private class HttpsRedirectFilter(scheme: String, securePort: Int) extends Filter { + + override def doFilter( + request: ServletRequest, + response: ServletResponse, + chain: FilterChain): Unit = { + if (request.isSecure) { + return + } + val httpRequest = request.asInstanceOf[HttpServletRequest] + val httpResponse = response.asInstanceOf[HttpServletResponse] + val httpsURI = createRedirectURI(scheme, securePort, httpRequest) + httpResponse.setContentLength(0) + httpResponse.sendRedirect(httpResponse.encodeRedirectURL(httpsURI)) + } + // Create a new URI from the arguments, handling IPv6 host encoding and default ports. + private def createRedirectURI(scheme: String, port: Int, request: HttpServletRequest): String = { + val server = request.getServerName + val redirectServer = if (server.contains(":") && !server.startsWith("[")) { + s"[$server]" + } else { + server + } + val authority = s"$redirectServer:$port" + val characterEncoding = if (request.getCharacterEncoding != null) { + request.getCharacterEncoding + } else { + // By default decoding the URI as "UTF-8" should be enough for SparkUI + "UTF-8" + } + // The request URL can be raw or encoded here. To avoid the request URL being + // encoded twice, let's decode it here. + val requestURI = decodeURL(request.getRequestURI, characterEncoding) + val queryString = decodeURL(request.getQueryString, characterEncoding) + new URI(scheme, authority, requestURI, queryString, null).toString + } + + private def decodeURL(url: String, encoding: String): String = { + if (url == null) { + null + } else { + URLDecoder.decode(url, encoding) + } + } } diff --git a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala index a23764f98d49..ca406b382020 100644 --- a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala +++ b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala @@ -20,7 +20,7 @@ package org.apache.spark.ui import java.util.Date import jakarta.servlet.http.{HttpServlet, HttpServletRequest, HttpServletResponse} -import org.eclipse.jetty.servlet.ServletContextHandler +import org.eclipse.jetty.ee10.servlet.ServletContextHandler import org.apache.spark.{SecurityManager, SparkConf, SparkContext} import org.apache.spark.internal.Logging diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala index 8644ab4c1e55..42fbb5a41934 100644 --- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala @@ -32,6 +32,7 @@ import scala.xml.transform.{RewriteRule, RuleTransformer} import jakarta.servlet.http.HttpServletRequest import jakarta.ws.rs.core.{MediaType, MultivaluedMap, Response} +import org.eclipse.jetty.server.Request import org.glassfish.jersey.internal.util.collection.MultivaluedStringMap import org.apache.spark.internal.Logging @@ -186,9 +187,9 @@ private[spark] object UIUtils extends Logging { } // Yarn has to go through a proxy so the base uri is provided and has to be on all links - def uiRoot(request: HttpServletRequest): String = { + def uiRoot(knoxBasePathGetter: String => String): String = { // Knox uses X-Forwarded-Context to notify the application the base path - val knoxBasePath = Option(request.getHeader("X-Forwarded-Context")) + val knoxBasePath = Option(knoxBasePathGetter("X-Forwarded-Context")) // SPARK-11484 - Use the proxyBase set by the AM, if not found then use env. sys.props.get("spark.ui.proxyBase") .orElse(sys.env.get("APPLICATION_WEB_PROXY_BASE")) @@ -196,6 +197,14 @@ private[spark] object UIUtils extends Logging { .getOrElse("") } + def uiRoot(request: HttpServletRequest): String = { + uiRoot(request.getHeader _) + } + + def uiRoot(request: Request): String = { + uiRoot(request.getHeaders.get: String => String) + } + def prependBaseUri( request: HttpServletRequest, basePath: String = "", diff --git a/core/src/main/scala/org/apache/spark/ui/WebUI.scala b/core/src/main/scala/org/apache/spark/ui/WebUI.scala index c0d0fb50ae36..7f8f2556dd08 100644 --- a/core/src/main/scala/org/apache/spark/ui/WebUI.scala +++ b/core/src/main/scala/org/apache/spark/ui/WebUI.scala @@ -25,7 +25,7 @@ import scala.xml.Node import jakarta.servlet.DispatcherType import jakarta.servlet.http.{HttpServlet, HttpServletRequest} -import org.eclipse.jetty.servlet.{FilterHolder, FilterMapping, ServletContextHandler, ServletHolder} +import org.eclipse.jetty.ee10.servlet.{FilterHolder, FilterMapping, ServletContextHandler, ServletHolder} import org.json4s.JsonAST.{JNothing, JValue} import org.apache.spark.{SecurityManager, SparkConf, SSLOptions} diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index 6babba0cd6d1..57e4f1236af5 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -60,7 +60,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.logging.log4j.{Level, LogManager} import org.apache.logging.log4j.core.LoggerContext import org.apache.logging.log4j.core.config.LoggerConfig -import org.eclipse.jetty.util.MultiException import org.slf4j.Logger import org.apache.spark.{SPARK_VERSION, _} @@ -2302,8 +2301,6 @@ private[spark] object Utils return true } isBindCollision(e.getCause) - case e: MultiException => - e.getThrowables.asScala.exists(isBindCollision) case e: NativeIoException => (e.getMessage != null && e.getMessage.matches("bind.*failed.*")) || isBindCollision(e.getCause) diff --git a/core/src/test/scala/org/apache/spark/deploy/history/ApplicationCacheSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/ApplicationCacheSuite.scala index d27c07c36c0d..f5968e383b05 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/ApplicationCacheSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/ApplicationCacheSuite.scala @@ -25,7 +25,7 @@ import scala.collection.mutable import com.codahale.metrics.Counter import jakarta.servlet.http.{HttpServletRequest, HttpServletResponse} import org.apache.hadoop.conf.Configuration -import org.eclipse.jetty.servlet.ServletContextHandler +import org.eclipse.jetty.ee10.servlet.ServletContextHandler import org.mockito.ArgumentMatchers.any import org.mockito.Mockito._ import org.mockito.invocation.InvocationOnMock diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala index 0a564f571521..3c09f956c189 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala @@ -654,17 +654,22 @@ abstract class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with val multiAttemptAppid = "local-1430917381535" val lastAttemptId = Some(2) val lastAttemptUrl = buildPageAttemptUrl(multiAttemptAppid, lastAttemptId) - Seq(None, Some(1), Some(2)).foreach { attemptId => - val url = buildPageAttemptUrl(multiAttemptAppid, attemptId) - val (code, location) = getRedirectUrl(url) - assert(code === 302, s"Unexpected status code $code for $url") - attemptId match { - case None => - assert(location.stripSuffix("/") === lastAttemptUrl.toString) - case _ => - assert(location.stripSuffix("/") === url.toString) - } - HistoryServerSuite.getUrl(new URI(location).toURL) + // If an application has multiple attempts, the path ends with the last attempt ID is the root + // of the context path of the application. + Seq((None, 302), (Some(1), 302), (Some(2), 301)).foreach { + case (attemptId, expectedCode) => + val url = buildPageAttemptUrl(multiAttemptAppid, attemptId) + val (code, location) = getRedirectUrl(url) + assert( + code === expectedCode, s"Unexpected status code $code for $url") + attemptId match { + case None => + assert(location.stripSuffix("/") === lastAttemptUrl.getPath) + case _ => + assert(location.stripSuffix("/") === url.getPath) + } + HistoryServerSuite.getUrl( + new URI(url.getProtocol, url.getAuthority, location, null, null).toURL) } } @@ -674,13 +679,13 @@ abstract class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with val url = buildPageAttemptUrl(oneAttemptAppId, None) val (code, location) = getRedirectUrl(url) - assert(code === 302, s"Unexpected status code $code for $url") - assert(location === url.toString + "/") + assert(code === 301, s"Unexpected status code $code for $url") + assert(location === url.getPath + "/") val url2 = buildPageAttemptUrl(multiAttemptAppid, None) val (code2, location2) = getRedirectUrl(url2) assert(code2 === 302, s"Unexpected status code $code2 for $url2") - assert(location2 === url2.toString + "/2/") + assert(location2 === url2.getPath + "/2/") } def getRedirectUrl(url: URL): (Int, String) = { @@ -738,7 +743,7 @@ abstract class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with conn.setInstanceFollowRedirects(false) conn.connect() assert(conn.getResponseCode === 302) - assert(conn.getHeaderField("Location") === s"http://$localhost:$port/") + assert(conn.getHeaderField("Location") === "/") } } diff --git a/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala index 7effeee3424b..a1ac2a0f09d9 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala @@ -18,8 +18,8 @@ package org.apache.spark.deploy.history import jakarta.servlet.http.HttpServletRequest -import org.eclipse.jetty.proxy.ProxyServlet -import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder} +import org.eclipse.jetty.ee10.proxy.ProxyServlet +import org.eclipse.jetty.ee10.servlet.{ServletContextHandler, ServletHolder} import org.openqa.selenium.WebDriver import org.scalatest.concurrent.Eventually._ import org.scalatest.matchers.must.Matchers diff --git a/core/src/test/scala/org/apache/spark/ui/UISuite.scala b/core/src/test/scala/org/apache/spark/ui/UISuite.scala index 6d12e88e8efa..5e458bf09c76 100644 --- a/core/src/test/scala/org/apache/spark/ui/UISuite.scala +++ b/core/src/test/scala/org/apache/spark/ui/UISuite.scala @@ -25,7 +25,7 @@ import scala.io.Source import jakarta.servlet._ import jakarta.servlet.http.{HttpServlet, HttpServletRequest, HttpServletResponse} -import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder} +import org.eclipse.jetty.ee10.servlet.{ServletContextHandler, ServletHolder} import org.eclipse.jetty.util.thread.QueuedThreadPool import org.mockito.Mockito.{mock, when} import org.scalatest.concurrent.Eventually._ @@ -234,7 +234,7 @@ class UISuite extends SparkFunSuite { val targetUri = URI.create(s"http://$localhost:4040") when(clientRequest.getScheme()).thenReturn("http") when(clientRequest.getHeader("host")).thenReturn(s"$localhost:8080") - when(clientRequest.getPathInfo()).thenReturn("/proxy/worker-id/jobs") + when(clientRequest.getPathInfo()).thenReturn("/worker-id/jobs") var newHeader = JettyUtils.createProxyLocationHeader(headerValue, clientRequest, targetUri) assert(newHeader.toString() === s"http://$localhost:8080/proxy/worker-id/jobs") headerValue = s"http://$localhost:4041/jobs" @@ -368,8 +368,11 @@ class UISuite extends SparkFunSuite { serverInfo.addHandler(redirect, securityMgr) // Test Jetty's built-in redirect to add the trailing slash to the context path. + // As of `Jetty 12`, when asking for the root of a servlet context without the trailing + // slash, status code 301 is returned. TestUtils.withHttpConnection(new URI(s"$serverAddr/ctx1").toURL) { conn => - assert(conn.getResponseCode() === HttpServletResponse.SC_FOUND) + conn.setInstanceFollowRedirects(false) + assert(conn.getResponseCode() === HttpServletResponse.SC_MOVED_PERMANENTLY) val location = Option(conn.getHeaderFields().get("Location")) .map(_.get(0)).orNull assert(location === s"$proxyRoot/ctx1/") @@ -400,7 +403,7 @@ class UISuite extends SparkFunSuite { } } - test("SPARK-45522: Jetty 10 and above shouuld return status code 302 with correct redirect url" + + test("SPARK-47086: Jetty 12 and above should return status code 301 with correct redirect url" + " when request URL ends with a context path without trailing '/'") { val proxyRoot = "https://proxy.example.com:443/prefix" val (conf, securityMgr, sslOptions) = sslDisabledConf() @@ -415,9 +418,10 @@ class UISuite extends SparkFunSuite { assert(TestUtils.httpResponseCode(new URI(urlStr + "/").toURL) === HttpServletResponse.SC_OK) // In the case of trailing slash, - // 302 should be return and the redirect URL shouuld be part of the header. - assert(TestUtils.redirectUrl(new URI(urlStr).toURL) === proxyRoot + "/ctx/"); - assert(TestUtils.httpResponseCode(new URI(urlStr).toURL) === HttpServletResponse.SC_FOUND) + // 301 should be return and the redirect URL should be part of the header. + assert(TestUtils.redirectUrl(new URI(urlStr).toURL) === proxyRoot + "/ctx/") + assert(TestUtils.httpResponseCode( + new URI(urlStr).toURL) === HttpServletResponse.SC_MOVED_PERMANENTLY) } finally { stopServer(serverInfo) } diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3 index 9bba9157054a..f22937cf186c 100644 --- a/dev/deps/spark-deps-hadoop-3-hive-2.3 +++ b/dev/deps/spark-deps-hadoop-3-hive-2.3 @@ -109,9 +109,9 @@ jackson-module-scala_2.13/2.20.1//jackson-module-scala_2.13-2.20.1.jar jakarta.activation-api/2.1.3//jakarta.activation-api-2.1.3.jar jakarta.annotation-api/2.1.1//jakarta.annotation-api-2.1.1.jar jakarta.inject-api/2.0.1//jakarta.inject-api-2.0.1.jar -jakarta.servlet-api/5.0.0//jakarta.servlet-api-5.0.0.jar +jakarta.servlet-api/6.0.0//jakarta.servlet-api-6.0.0.jar jakarta.validation-api/3.0.2//jakarta.validation-api-3.0.2.jar -jakarta.ws.rs-api/3.0.0//jakarta.ws.rs-api-3.0.0.jar +jakarta.ws.rs-api/3.1.0//jakarta.ws.rs-api-3.1.0.jar jakarta.xml.bind-api/4.0.2//jakarta.xml.bind-api-4.0.2.jar janino/3.1.9//janino-3.1.9.jar java-diff-utils/4.16//java-diff-utils-4.16.jar @@ -125,15 +125,15 @@ jaxb-runtime/4.0.5//jaxb-runtime-4.0.5.jar jcl-over-slf4j/2.0.17//jcl-over-slf4j-2.0.17.jar jdo-api/3.0.1//jdo-api-3.0.1.jar jdom2/2.0.6//jdom2-2.0.6.jar -jersey-client/3.0.18//jersey-client-3.0.18.jar -jersey-common/3.0.18//jersey-common-3.0.18.jar -jersey-container-servlet-core/3.0.18//jersey-container-servlet-core-3.0.18.jar -jersey-container-servlet/3.0.18//jersey-container-servlet-3.0.18.jar -jersey-hk2/3.0.18//jersey-hk2-3.0.18.jar -jersey-server/3.0.18//jersey-server-3.0.18.jar +jersey-client/3.1.11//jersey-client-3.1.11.jar +jersey-common/3.1.11//jersey-common-3.1.11.jar +jersey-container-servlet-core/3.1.11//jersey-container-servlet-core-3.1.11.jar +jersey-container-servlet/3.1.11//jersey-container-servlet-3.1.11.jar +jersey-hk2/3.1.11//jersey-hk2-3.1.11.jar +jersey-server/3.1.11//jersey-server-3.1.11.jar jettison/1.5.4//jettison-1.5.4.jar -jetty-util-ajax/11.0.26//jetty-util-ajax-11.0.26.jar -jetty-util/11.0.26//jetty-util-11.0.26.jar +jetty-util-ajax/12.1.4//jetty-util-ajax-12.1.4.jar +jetty-util/12.1.4//jetty-util-12.1.4.jar jjwt-api/0.12.6//jjwt-api-0.12.6.jar jjwt-impl/0.12.6//jjwt-impl-0.12.6.jar jjwt-jackson/0.12.6//jjwt-jackson-0.12.6.jar diff --git a/pom.xml b/pom.xml index a01a41909d2e..2cc9c1d95c1c 100644 --- a/pom.xml +++ b/pom.xml @@ -144,8 +144,8 @@ 1.16.0 2.2.1 shaded-protobuf - 11.0.26 - 5.0.0 + 12.1.4 + 6.0.0 4.0.1 0.10.0 @@ -202,7 +202,7 @@ 1.0.3 2.11.0 3.1.9 - 3.0.18 + 3.1.11 2.14.0 3.0.0 2.2.11 @@ -561,20 +561,20 @@ provided - org.eclipse.jetty - jetty-servlet + org.eclipse.jetty.ee10 + jetty-ee10-servlet ${jetty.version} provided - org.eclipse.jetty - jetty-servlets + org.eclipse.jetty.ee10 + jetty-ee10-servlets ${jetty.version} provided - org.eclipse.jetty - jetty-proxy + org.eclipse.jetty.ee10 + jetty-ee10-proxy ${jetty.version} provided @@ -597,8 +597,8 @@ provided - org.eclipse.jetty - jetty-plus + org.eclipse.jetty.ee10 + jetty-ee10-plus ${jetty.version} provided @@ -609,8 +609,8 @@ provided - org.eclipse.jetty - jetty-webapp + org.eclipse.jetty.ee10 + jetty-ee10-webapp ${jetty.version} provided diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index b7239c4c6160..4f5a6f1511ac 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -35,6 +35,10 @@ object MimaExcludes { // Exclude rules for 4.2.x from 4.1.0 lazy val v42excludes = v41excludes ++ Seq( + // [SPARK-47086][BUILD][CORE][WEBUI] Upgrade Jetty to 12.1.4 + ProblemFilters.exclude[MissingTypesProblem]("org.apache.spark.ui.ProxyRedirectHandler$ResponseWrapper"), + ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ui.ProxyRedirectHandler#ResponseWrapper.sendRedirect"), + ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.ui.ProxyRedirectHandler#ResponseWrapper.this") ) // Exclude rules for 4.1.x from 4.0.0 diff --git a/repl/pom.xml b/repl/pom.xml index f0d0dd396389..2eb732c1c546 100644 --- a/repl/pom.xml +++ b/repl/pom.xml @@ -119,8 +119,8 @@ jetty-server - org.eclipse.jetty - jetty-plus + org.eclipse.jetty.ee10 + jetty-ee10-plus org.eclipse.jetty diff --git a/resource-managers/yarn/pom.xml b/resource-managers/yarn/pom.xml index 30169ffa9672..3d4d441dfcf9 100644 --- a/resource-managers/yarn/pom.xml +++ b/resource-managers/yarn/pom.xml @@ -108,8 +108,8 @@ jetty-server - org.eclipse.jetty - jetty-plus + org.eclipse.jetty.ee10 + jetty-ee10-plus org.eclipse.jetty @@ -120,12 +120,12 @@ jetty-http - org.eclipse.jetty - jetty-servlet + org.eclipse.jetty.ee10 + jetty-ee10-servlet - org.eclipse.jetty - jetty-servlets + org.eclipse.jetty.ee10 + jetty-ee10-servlets diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/AmIpFilterSuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/AmIpFilterSuite.scala index aabf35d66e6b..f9bd0ab844b9 100644 --- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/AmIpFilterSuite.scala +++ b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/AmIpFilterSuite.scala @@ -29,8 +29,8 @@ import scala.jdk.CollectionConverters._ import jakarta.servlet.{FilterChain, FilterConfig, ServletContext, ServletException, ServletOutputStream, ServletRequest, ServletResponse} import jakarta.servlet.http.{Cookie, HttpServlet, HttpServletRequest, HttpServletResponse} import jakarta.ws.rs.core.MediaType +import org.eclipse.jetty.ee10.servlet.{ServletContextHandler, ServletHolder} import org.eclipse.jetty.server.{Server, ServerConnector} -import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder} import org.eclipse.jetty.util.thread.QueuedThreadPool import org.mockito.Mockito.{mock, when} import org.scalatest.concurrent.Eventually._ @@ -272,10 +272,6 @@ class AmIpFilterSuite extends SparkFunSuite { override def encodeRedirectURL(url: String): String = url - override def encodeUrl(url: String): String = null - - override def encodeRedirectUrl(url: String): String = null - @throws[IOException] override def sendError(sc: Int, msg: String): Unit = {} @@ -284,8 +280,6 @@ class AmIpFilterSuite extends SparkFunSuite { override def setStatus(status: Int): Unit = this.status = status - override def setStatus(sc: Int, sm: String): Unit = {} - override def getStatus: Int = 0 override def setContentType(contentType: String): Unit = this.contentType = contentType diff --git a/sql/core/pom.xml b/sql/core/pom.xml index a5b5c399d4fc..ab6a8f8182e6 100644 --- a/sql/core/pom.xml +++ b/sql/core/pom.xml @@ -159,8 +159,8 @@ parquet-hadoop - org.eclipse.jetty - jetty-servlet + org.eclipse.jetty.ee10 + jetty-ee10-servlet com.fasterxml.jackson.core diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml index b70180ab753a..58447cfa4ece 100644 --- a/sql/hive-thriftserver/pom.xml +++ b/sql/hive-thriftserver/pom.xml @@ -91,8 +91,8 @@ provided - org.eclipse.jetty - jetty-servlet + org.eclipse.jetty.ee10 + jetty-ee10-servlet provided diff --git a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java index 9592bffcf1bf..1ada2bdb0bca 100644 --- a/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java +++ b/sql/hive-thriftserver/src/main/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java @@ -41,8 +41,8 @@ import org.eclipse.jetty.server.ConnectionFactory; import org.eclipse.jetty.server.HttpConnectionFactory; import org.eclipse.jetty.server.ServerConnector; -import org.eclipse.jetty.servlet.ServletContextHandler; -import org.eclipse.jetty.servlet.ServletHolder; +import org.eclipse.jetty.ee10.servlet.ServletContextHandler; +import org.eclipse.jetty.ee10.servlet.ServletHolder; import org.eclipse.jetty.util.ssl.SslContextFactory; import org.eclipse.jetty.util.thread.ExecutorThreadPool; import org.eclipse.jetty.util.thread.ScheduledExecutorScheduler; diff --git a/streaming/pom.xml b/streaming/pom.xml index a8fd54bbf2b8..1822d4699f27 100644 --- a/streaming/pom.xml +++ b/streaming/pom.xml @@ -76,8 +76,8 @@ jetty-server - org.eclipse.jetty - jetty-plus + org.eclipse.jetty.ee10 + jetty-ee10-plus org.eclipse.jetty @@ -88,12 +88,12 @@ jetty-http - org.eclipse.jetty - jetty-servlet + org.eclipse.jetty.ee10 + jetty-ee10-servlet - org.eclipse.jetty - jetty-servlets + org.eclipse.jetty.ee10 + jetty-ee10-servlets