Upgrade HtmlUnit to 2.55.0 and Jetty to 9.4.44

Several old tests previously not able to run in HtmlUnit are now
enabled, and two tests are disabled, linked to the appropriate bug
report.

The jaspi security/authentication module has been disabled in the dev
mode server. It is no longer included in the base jetty install due to
licensing reasons, and is disabled by default here for the same reason.

Many small changes were necessary across various parts of testing and
dev mode infrastructure:
 * HtmlUnit has introduced a proxy for window and document, legacy dev
   mode needs to treat those objects as if they were the real instance.
 * HtmlUnit has additional protection to avoid letting a page interact
   with a real Java object. This had to be relaxed in order to let
   legacy dev mode tests continue to work.
 * The custom classloader used to run a webapp within dev mode now
   overrides loadClass instead of findClass, to correctly delegate to
   Jetty's own implementation to handle startup issues. Additionally,
   isSystemClass and isServerClass are relocated and deprecated.

Bug: gwtproject#9720
Bug-Link: gwtproject#9720
Co-Authored-By: Mihai Stanciu <[email protected]>
Change-Id: Ie14cb33cb0edea57c121de943f2964a8cf0540ee

dev/build.xml

@@ -79,44 +79,46 @@
           <include name="colt/colt-1.2.jar"/>
           <include name="eclipse/org.eclipse.jdt.core_3.17.0.v20190306-2240.jar"/>
           <include name="eclipse/jdtCompilerAdapter_3.17.0.v20190306-2240.jar"/>
-          <include name="objectweb/asm-7.1/asm-7.1.jar"/>
-          <include name="objectweb/asm-7.1/asm-commons-7.1.jar"/>
-          <include name="objectweb/asm-7.1/asm-util-7.1.jar"/>
+          <include name="objectweb/asm-9.2/asm-9.2.jar"/>
+          <include name="objectweb/asm-9.2/asm-commons-9.2.jar"/>
+          <include name="objectweb/asm-9.2/asm-util-9.2.jar"/>
           <include name="guava/guava-19.0/guava-19.0-rebased.jar"/>
           <include name="icu4j/63.1/icu4j.jar"/>
-          <include name="jetty/jetty-9.2.14.v20151106/jetty-all-9.2.14.v20151106.jar"/>
+          <include name="jetty/jetty-9.4.44.v20210927/jetty-all-9.4.44.v20210927.jar"/>
           <include name="gson/gson-2.6.2.jar"/>
           <include name="jscomp/20160315/sourcemap-rebased.jar"/>
           <include name="jsr305/jsr305.jar"/>
           <include name="protobuf/protobuf-2.5.0/protobuf-java-rebased-2.5.0.jar"/>
           <!-- dependencies needed for JSP support in DevMode: BEGIN -->
-          <include name="jetty/jetty-9.2.14.v20151106/mortbay-apache-jsp-8.0.9.M3.jar"/>
-          <include name="jetty/jetty-9.2.14.v20151106/mortbay-apache-el-8.0.9.M3.jar"/>
-          <include name="jetty/jetty-9.2.14.v20151106/jetty-apache-jsp-9.2.14.v20151106.jar"/>
+          <include name="jetty/jetty-9.4.44.v20210927/mortbay-apache-jsp-8.5.70.jar"/>
+          <include name="jetty/jetty-9.4.44.v20210927/mortbay-apache-el-8.5.70.jar"/>
+          <include name="jetty/jetty-9.4.44.v20210927/jetty-apache-jsp-9.4.44.v20210927.jar"/>
           <!-- dependencies needed for JSP support in DevMode: END -->
           <include name="tomcat/tomcat-servlet-api-8.0.28.jar"/>
           <include name="tomcat/tomcat-websocket-api-8.0.28.jar"/>
           <include name="tomcat/tomcat-annotations-api-8.0.28.jar"/>
           <include name="apache/commons/commons-collections-3.2.2.jar"/>
           <!-- htmlunit dependencies not already included: BEGIN -->
-          <include name="apache/http/httpclient-4.5.1.jar"/>
-          <include name="apache/http/httpcore-4.4.4.jar"/>
-          <include name="apache/http/httpmime-4.5.1.jar"/>
+          <include name="apache/http/httpclient-4.5.13.jar"/>
+          <include name="apache/http/httpcore-4.4.13.jar"/>
+          <include name="apache/http/httpmime-4.5.13.jar"/>
           <include name="apache/james/apache-mime4j-0.6.jar"/>
           <include name="apache/commons/commons-codec-1.10.jar"/>
           <include name="apache/commons/commons-io-2.4.jar"/>
           <include name="apache/commons/commons-lang3-3.4.jar"/>
           <include name="apache/commons/commons-logging-1.2.jar"/>
-          <include name="cssparser/cssparser-0.9.18.jar"/>
-         <include name="htmlunit/htmlunit-2.19/htmlunit-2.19.jar"/>
-         <include name="htmlunit/htmlunit-2.19/htmlunit-core-js-2.15.jar"/>
-          <include name="nekohtml/nekohtml-1.9.22.jar"/>
+          <include name="htmlunit/htmlunit-2.55.0/htmlunit-cssparser-1.10.0.jar"/>
+          <include name="htmlunit/htmlunit-2.55.0/htmlunit-2.55.0.jar"/>
+          <include name="htmlunit/htmlunit-2.55.0/htmlunit-core-js-2.55.0.jar"/>
+          <include name="htmlunit/htmlunit-2.55.0/neko-htmlunit-2.55.0.jar"/>
+          <include name="htmlunit/htmlunit-2.55.0/salvation2-3.0.0.jar"/>
           <include name="cup/java-cup-11a.jar"/>
           <include name="xalan/xalan-2.7.1-nocup.jar"/>
           <include name="xerces/xerces-2_11_0/serializer-2.7.1.jar"/>
           <include name="xerces/xerces-2_11_0/xercesImpl-2.11.0.jar"/>
           <include name="xerces/xerces-2_11_0/xml-apis-1.4.01.jar"/>
           <include name="w3c/sac/sac-1.3.jar"/>
+          <include name="brotli/dec-0.1.2.jar"/>
           <!-- htmlunit dependencies not already included: END -->
           <include name="json/android-sdk-19.1/json-android-rebased.jar"/>
         </fileset>
@@ -131,9 +133,9 @@
             <provider classname="org.eclipse.jetty.websocket.jsr356.server.deploy.WebSocketServerContainerInitializer"/>
             <provider classname="org.eclipse.jetty.apache.jsp.JettyJasperInitializer"/>
           </service>
-          <zipfileset src="${gwt.tools.lib}/objectweb/asm-7.1/asm-7.1.jar"/>
-          <zipfileset src="${gwt.tools.lib}/objectweb/asm-7.1/asm-commons-7.1.jar"/>
-          <zipfileset src="${gwt.tools.lib}/objectweb/asm-7.1/asm-util-7.1.jar"/>
+          <zipfileset src="${gwt.tools.lib}/objectweb/asm-9.2/asm-9.2.jar"/>
+          <zipfileset src="${gwt.tools.lib}/objectweb/asm-9.2/asm-commons-9.2.jar"/>
+          <zipfileset src="${gwt.tools.lib}/objectweb/asm-9.2/asm-util-9.2.jar"/>
           <zipfileset src="${gwt.tools.lib}/apache/tapestry-util-text-4.0.2.jar"/>
           <zipfileset src="${gwt.tools.lib}/apache/ant-zipscanner/ant-zipscanner-1.6.5-1-rebased.jar"/>
           <zipfileset src="${gwt.tools.lib}/colt/colt-1.2.jar"/>
@@ -144,42 +146,50 @@
           <zipfileset src="${gwt.tools.lib}/guava/guava-19.0/guava-19.0-rebased.jar"/>
           <zipfileset src="${gwt.tools.lib}/icu4j/63.1/icu4j.jar"/>
           <zipfileset
-              src="${gwt.tools.lib}/jetty/jetty-9.2.14.v20151106/jetty-all-9.2.14.v20151106.jar"/>
+              src="${gwt.tools.lib}/jetty/jetty-9.4.44.v20210927/jetty-all-9.4.44.v20210927.jar">
+            <!--
+            Don't add jetty auth implementation from jetty-jaspi, as we would also need to package jetty's
+            javax.security.auth.message implementation as well as geronimo-jaspi.
+             -->
+            <exclude name="META-INF/services/org.eclipse.jetty.security.Authenticator$Factory"/>
+          </zipfileset>
           <zipfileset src="${gwt.tools.lib}/gson/gson-2.6.2.jar"/>
           <zipfileset src="${gwt.tools.lib}/jscomp/20160315/sourcemap-rebased.jar"/>
           <zipfileset src="${gwt.tools.lib}/jsr305/jsr305.jar"/>
           <zipfileset
               src="${gwt.tools.lib}/protobuf/protobuf-2.5.0/protobuf-java-rebased-2.5.0.jar"/>
           <!-- dependencies needed for JSP support in DevMode: BEGIN -->
-          <zipfileset src="${gwt.tools.lib}/jetty/jetty-9.2.14.v20151106/mortbay-apache-jsp-8.0.9.M3.jar"/>
-          <zipfileset src="${gwt.tools.lib}/jetty/jetty-9.2.14.v20151106/mortbay-apache-el-8.0.9.M3.jar"/>
-          <zipfileset src="${gwt.tools.lib}/jetty/jetty-9.2.14.v20151106/jetty-apache-jsp-9.2.14.v20151106.jar"/>
+          <zipfileset src="${gwt.tools.lib}/jetty/jetty-9.4.44.v20210927/mortbay-apache-jsp-8.5.70.jar"/>
+          <zipfileset src="${gwt.tools.lib}/jetty/jetty-9.4.44.v20210927/mortbay-apache-el-8.5.70.jar"/>
+          <zipfileset src="${gwt.tools.lib}/jetty/jetty-9.4.44.v20210927/jetty-apache-jsp-9.4.44.v20210927.jar"/>
           <!-- dependencies needed for JSP support in DevMode: END -->
           <zipfileset src="${gwt.tools.lib}/tomcat/tomcat-servlet-api-8.0.28.jar"/>
           <zipfileset src="${gwt.tools.lib}/tomcat/tomcat-websocket-api-8.0.28.jar"/>
           <zipfileset src="${gwt.tools.lib}/tomcat/tomcat-annotations-api-8.0.28.jar"/>
           <zipfileset
               src="${gwt.tools.lib}/apache/commons/commons-collections-3.2.2.jar"/>
           <!-- htmlunit dependencies not already included: BEGIN -->
-          <zipfileset src="${gwt.tools.lib}/apache/http/httpclient-4.5.1.jar"/>
-          <zipfileset src="${gwt.tools.lib}/apache/http/httpcore-4.4.4.jar"/>
-          <zipfileset src="${gwt.tools.lib}/apache/http/httpmime-4.5.1.jar"/>
+          <zipfileset src="${gwt.tools.lib}/apache/http/httpclient-4.5.13.jar"/>
+          <zipfileset src="${gwt.tools.lib}/apache/http/httpcore-4.4.13.jar"/>
+          <zipfileset src="${gwt.tools.lib}/apache/http/httpmime-4.5.13.jar"/>
           <zipfileset src="${gwt.tools.lib}/apache/james/apache-mime4j-0.6.jar"/>
           <zipfileset src="${gwt.tools.lib}/apache/commons/commons-codec-1.10.jar"/>
           <zipfileset src="${gwt.tools.lib}/apache/commons/commons-io-2.4.jar"/>
           <zipfileset src="${gwt.tools.lib}/apache/commons/commons-lang3-3.4.jar"/>
           <zipfileset src="${gwt.tools.lib}/apache/commons/commons-logging-1.2.jar"/>
-          <zipfileset src="${gwt.tools.lib}/cssparser/cssparser-0.9.18.jar"/>
-          <zipfileset src="${gwt.tools.lib}/htmlunit/htmlunit-2.19/htmlunit-2.19.jar"/>
+          <zipfileset src="${gwt.tools.lib}/htmlunit/htmlunit-2.55.0/htmlunit-cssparser-1.10.0.jar"/>
+          <zipfileset src="${gwt.tools.lib}/htmlunit/htmlunit-2.55.0/htmlunit-2.55.0.jar"/>
           <zipfileset
-              src="${gwt.tools.lib}/htmlunit/htmlunit-2.19/htmlunit-core-js-2.15.jar"/>
-          <zipfileset src="${gwt.tools.lib}/nekohtml/nekohtml-1.9.22.jar"/>
+              src="${gwt.tools.lib}/htmlunit/htmlunit-2.55.0/htmlunit-core-js-2.55.0.jar"/>
+          <zipfileset src="${gwt.tools.lib}/htmlunit/htmlunit-2.55.0/neko-htmlunit-2.55.0.jar"/>
+          <zipfileset src="${gwt.tools.lib}/htmlunit/htmlunit-2.55.0/salvation2-3.0.0.jar"/>
           <zipfileset src="${gwt.tools.lib}/cup/java-cup-11a.jar"/>
           <zipfileset src="${gwt.tools.lib}/xalan/xalan-2.7.1-nocup.jar"/>
           <zipfileset src="${gwt.tools.lib}/xerces/xerces-2_11_0/serializer-2.7.1.jar"/>
           <zipfileset src="${gwt.tools.lib}/xerces/xerces-2_11_0/xercesImpl-2.11.0.jar"/>
           <zipfileset src="${gwt.tools.lib}/xerces/xerces-2_11_0/xml-apis-1.4.01.jar"/>
           <zipfileset src="${gwt.tools.lib}/w3c/sac/sac-1.3.jar"/>
+          <zipfileset src="${gwt.tools.lib}/brotli/dec-0.1.2.jar"/>
           <!-- htmlunit dependencies not already included: END -->
           <zipfileset src="${gwt.tools.lib}/json/android-sdk-19.1/json-android-rebased.jar"/>
         </gwt.jar>
@@ -226,9 +236,9 @@
       <classpath>
         <pathelement location="${gwt.tools.lib}/apache/ant-zipscanner/ant-zipscanner-1.6.5-1-rebased.jar"/>
         <pathelement location="${gwt.tools.lib}/colt/colt-1.2.jar"/>
-        <pathelement location="${gwt.tools.lib}/objectweb/asm-7.1/asm-7.1.jar"/>
-        <pathelement location="${gwt.tools.lib}/objectweb/asm-7.1/asm-commons-7.1.jar"/>
-        <pathelement location="${gwt.tools.lib}/objectweb/asm-7.1/asm-util-7.1.jar"/>
+        <pathelement location="${gwt.tools.lib}/objectweb/asm-9.2/asm-9.2.jar"/>
+        <pathelement location="${gwt.tools.lib}/objectweb/asm-9.2/asm-commons-9.2.jar"/>
+        <pathelement location="${gwt.tools.lib}/objectweb/asm-9.2/asm-util-9.2.jar"/>
         <pathelement
             location="${gwt.tools.lib}/apache/commons/commons-collections-3.2.2.jar"/>
         <pathelement

dev/codeserver/java/com/google/gwt/dev/codeserver/WebServer.java

@@ -123,7 +123,6 @@ void start(final TreeLogger logger) throws UnableToCompleteException {
     connector.setHost(bindAddress);
     connector.setPort(port);
     connector.setReuseAddress(false);
-    connector.setSoLingerTime(0);
     newServer.addConnector(connector);
 
     ServletContextHandler newHandler = new ServletContextHandler(ServletContextHandler.SESSIONS);

dev/core/src/com/google/gwt/dev/shell/HostedModePluginObject.java

@@ -17,16 +17,17 @@
 
 import com.google.gwt.core.ext.TreeLogger;
 
+import com.gargoylesoftware.htmlunit.WebClient;
 import com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine;
 import com.gargoylesoftware.htmlunit.javascript.host.Window;
-import com.gargoylesoftware.htmlunit.javascript.host.WindowProxy;
 
 import net.sourceforge.htmlunit.corejs.javascript.Context;
 import net.sourceforge.htmlunit.corejs.javascript.Function;
 import net.sourceforge.htmlunit.corejs.javascript.Scriptable;
 import net.sourceforge.htmlunit.corejs.javascript.ScriptableObject;
 
 import java.io.IOException;
+import java.util.Collections;
 
 /**
  * HTMLUnit object that represents the hosted-mode plugin.
@@ -122,11 +123,10 @@ public Object call(Context context, Scriptable scope, Scriptable thisObj,
             + " init: expected 1, got " + args.length);
       }
       try {
-        window = ((WindowProxy) args[0]).getDelegee();
+        window = (Window) args[0];
         return init(VERSION);
       } catch (ClassCastException e) {
-        throw Context.reportRuntimeError("Incorrect parameter types for "
-            + " initt: expected String");
+        throw Context.reportRuntimeError("Incorrect parameter types for init: expected Window");
       }
     }
 
@@ -149,6 +149,7 @@ public String getClassName() {
   private Scriptable initMethod;
   private Window window;
   private final JavaScriptEngine jsEngine;
+  private final WebClient webClient;
   private final TreeLogger logger;
 
   private BrowserChannelClient browserChannelClient;
@@ -157,9 +158,12 @@ public String getClassName() {
    * Creates a HostedModePluginObject with the passed-in JavaScriptEngine.
    *
    * @param jsEngine The JavaScriptEngine.
+   * @param webClient The WebClient being tested.
+   * @param logger A logger instance to notify the user of errors.
    */
-  public HostedModePluginObject(JavaScriptEngine jsEngine, TreeLogger logger) {
+  public HostedModePluginObject(JavaScriptEngine jsEngine, WebClient webClient, TreeLogger logger) {
     this.jsEngine = jsEngine;
+    this.webClient = webClient;
     this.logger = logger;
   }
 
@@ -184,9 +188,17 @@ public boolean connect(String url, String sessionKey, String address,
     }
     // TODO: add whitelist and default-port support?
 
+    // We know that legacy dev mode is running, we need to tell HtmlUnit that it is safe
+    // to permit plain Java objects to leak into JS - the JavaObject type will return a
+    // Object[] with a success boolean and a value, and HtmlUnit will guard against this.
+    // The simplest way to do that here is to mark java.lang.Object as the java equivalent
+    // of some JS type - the name of the type doesn't matter.
+    webClient.setActiveXObjectMap(Collections.singletonMap(
+            "GwtLegacyDevModeExceptionOrReturnValue", "java.lang.Object"));
+
     try {
       HtmlUnitSessionHandler htmlUnitSessionHandler = new HtmlUnitSessionHandler(
-          window, jsEngine);
+          window, jsEngine, webClient);
       browserChannelClient = new BrowserChannelClient(addressParts, url,
           sessionKey, module, version, htmlUnitSessionHandler);
       htmlUnitSessionHandler.setSessionData(new SessionData(

dev/core/src/com/google/gwt/dev/shell/HtmlUnitSessionHandler.java

@@ -25,6 +25,7 @@
 
 import com.gargoylesoftware.htmlunit.ScriptException;
 import com.gargoylesoftware.htmlunit.ScriptResult;
+import com.gargoylesoftware.htmlunit.WebClient;
 import com.gargoylesoftware.htmlunit.WebWindow;
 import com.gargoylesoftware.htmlunit.html.HtmlPage;
 import com.gargoylesoftware.htmlunit.javascript.JavaScriptEngine;
@@ -108,9 +109,11 @@ public String getClassName() {
   private final ToStringMethod toStringMethod = new ToStringMethod();
 
   private final Window window;
+  private final WebClient webClient;
 
-  HtmlUnitSessionHandler(Window window, JavaScriptEngine jsEngine) {
+  HtmlUnitSessionHandler(Window window, JavaScriptEngine jsEngine, WebClient webClient) {
     this.window = window;
+    this.webClient = webClient;
     logger.setMaxDetail(TreeLogger.ERROR);
     this.jsEngine = jsEngine;
     htmlPage = (HtmlPage) this.window.getWebWindow().getEnclosedPage();
@@ -161,7 +164,7 @@ public Object getToStringTearOff(Context jsContext) {
   @Override
   public String getUserAgent() {
     return "HtmlUnit-"
-        + jsEngine.getWebClient().getBrowserVersion().getUserAgent();
+        + webClient.getBrowserVersion().getUserAgent();
   }
 
   @SuppressWarnings("unchecked")
@@ -286,6 +289,15 @@ public Value makeValueFromJsval(Context jsContext, Object value) {
       return returnVal;
     }
     if (value instanceof Scriptable) {
+      if (value instanceof SimpleScriptableProxy) {
+        // HtmlUnit will return proxies to java for the window/document objects,
+        // so that those objects can work after navigating away from the page.
+        // However, GWTTestCase operates inside a single page session, so we
+        // can unwrap these proxies to get the real instance. Without doing
+        // this, the refToJsObject mapping would indicate that an object might
+        // not equal itself
+        value = ((SimpleScriptableProxy<?>) value).getDelegee();
+      }
       if (value instanceof ScriptableObject) {
         /*
          * HACK: check for native types like NativeString. NativeString is