Merge branch 'main' into feature/SOLR-17602/per-module-dependency-loc…

…king

# Conflicts:
#	dev-tools/scripts/smokeTestRelease.py
#	versions.lock

.github/workflows/bin-solr-test.yml

@@ -19,7 +19,7 @@ jobs:
     runs-on: ubuntu-latest
 
     env:
-      DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+      DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
 
     steps:
     # Setup

.github/workflows/docker-test.yml

@@ -21,7 +21,7 @@ jobs:
     env:
       SOLR_DOCKER_IMAGE_REPO: github-pr/solr
       SOLR_DOCKER_IMAGE_TAG: ${{github.event.number}}
-      DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+      DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
 
     steps:
     # Setup

.github/workflows/gradle-precommit.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
 
     env:
-      DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+      DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
 
     steps:
     # Setup

.github/workflows/solrj-test.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
 
     env:
-      DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+      DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
 
     steps:
     # Setup

build-tools/build-infra/src/main/java/org/apache/lucene/gradle/Checksum.java

@@ -47,7 +47,7 @@
 public class Checksum extends DefaultTask {
   private FileCollection files;
   private File outputDir;
-  private Algorithm algorithm = Checksum.Algorithm.SHA512;
+  private Algorithm algorithm;
 
   public enum Algorithm {
     MD5(new DigestUtils(DigestUtils.getMd5Digest())),
@@ -68,7 +68,7 @@ public String getExtension() {
 
   public Checksum() {
     outputDir = new File(getProject().getBuildDir(), "checksums");
-    algorithm = Algorithm.SHA256;
+    algorithm = Checksum.Algorithm.SHA512;
   }
 
   @InputFiles

build.gradle

@@ -121,10 +121,6 @@ ext {
 apply from: file('gradle/conventions.gradle')
 apply from: file('gradle/generation/local-settings.gradle')
 
-// Ant-compatibility layer: apply folder layout early so that
-// evaluation of other scripts doesn't need to be deferred.
-apply from: file('gradle/ant-compat/solr.folder-layout.gradle')
-
 // Set up defaults and configure aspects for certain modules or functionality
 // (java, tests)
 apply from: file('gradle/java/folder-layout.gradle')
@@ -187,13 +183,6 @@ apply from: file('gradle/help.gradle')
 // Configures development for joint Lucene/ Solr composite build.
 apply from: file('gradle/lucene-dev/lucene-dev-repo-composite.gradle')
 
-// Ant-compatibility layer. ALL OF THESE SHOULD BE GONE at some point. They are
-// here so that we can coexist with current ant build but they are indicative
-// of potential problems with the build conventions, dependencies, etc.
-apply from: file('gradle/ant-compat/misc.gradle')
-
-apply from: file('gradle/ant-compat/artifact-naming.gradle')
-
 apply from: file('gradle/documentation/pull-lucene-javadocs.gradle')
 apply from: file('gradle/documentation/documentation.gradle')
 apply from: file('gradle/documentation/changes-to-html.gradle')

dev-docs/FAQ.adoc

@@ -95,7 +95,7 @@ If you don't yet have an account, you have to ask for one in the 'users' or 'dev
 === Where can I find information about test history?
 
 * http://fucit.org/solr-jenkins-reports/failure-report.html
-* https://ge.apache.org/scans/tests?search.relativeStartTime=P90D&search.rootProjectNames=solr*
+* https://develocity.apache.org/scans/tests?search.relativeStartTime=P90D&search.rootProjectNames=solr*
 * https://lists.apache.org[Solr mailing list archives especially builds]
 
 === How can I build the JavaDoc's and the Reference Guide?

dev-docs/v2-api-conventions.adoc

@@ -1,3 +1,4 @@
+= API Design
 == HTTP Paths
 
 Where possible, each v2 API is given an HTTP path that reflects the resource type and/or name most relevant to its functionality.
@@ -69,8 +70,8 @@ For use within the v2 API, the four "popular" HTTP methods have the following se
 == Errors
 
 v2 APIs should be consistent in how they report errors.  Throwing a `SolrException` will convey
-1.the error code as the HTTP response status code, as `responseHeader.status` and as `error.code`, and
-1.the error message as `error.msg`.
+1. The error code as the HTTP response status code, as `responseHeader.status` and as `error.code`, and
+2. The error message as `error.msg`.
 
 API calls that reference a specific resource (e.g. `specificCollName`, `specificAliasName`, `specificPropertyName` and others per the above list) that do not exist should return `SolrException.ErrorCode.NOT_FOUND` (HTTP 404).
 
@@ -82,3 +83,23 @@ Often these operations were initially conceived of as procedural "commands" and
 
 Solr's v2 API currently accommodates these "command" APIs by appending the command name (often a verb like "unload", "reload", or "split") onto the otherwise "resource"-based path.
 For example: Solr's core "unload" command uses the API `POST /api/cores/specificCoreName/unload`.
+
+= JAX-RS Implementation Conventions
+
+== Streaming
+
+Solr has a number of APIs that return binary file data or other arbitrary content, such as the "replication" APIs used to pull index files from other cores.
+Please use the following conventions when implementing similar endpoints:
+1. `@Operation` annotations use an "extension property" to indicate to codegen tools that the API output is "raw" or untyped.  For example:
++
+```
+  @Operation(
+      summary = "Return the data stored in a specified ZooKeeper node",
+      tags = {"zookeeper-read"},
+      extensions = {
+        @Extension(properties = {@ExtensionProperty(name = RAW_OUTPUT_PROPERTY, value = "true")})
+      })
+```
+2. Interface methods should return a type that implements the JAX-RS `StreamingOutput` interface.
+
+See the `fetchFile()` method in `ReplicationApis.java` for a concrete example.

dev-tools/scripts/releaseWizard.py

@@ -68,7 +68,7 @@
 from scriptutil import BranchType, Version, download, run
 
 # Solr-to-Java version mapping
-java_versions = {6: 8, 7: 8, 8: 8, 9: 11, 10: 11}
+java_versions = {6: 8, 7: 8, 8: 8, 9: 11, 10: 21}
 editor = None
 state = None
 templates = None
@@ -1353,7 +1353,7 @@ def main():
 
     state.save()
 
-    # Smoketester requires JAVA11_HOME to point to Java11
+    # Smoketester requires JAVA21_HOME to point to Java21
     os.environ['JAVA_HOME'] = state.get_java_home()
     os.environ['JAVACMD'] = state.get_java_cmd()
 

dev-tools/scripts/releaseWizard.yaml

@@ -226,7 +226,7 @@ groups:
       You will need these tools:
 
       * Python v3.4 or later, with dependencies listed in requirements.txt
-      * Java 11 in $JAVA11_HOME
+      * Java 21 in $JAVA21_HOME
       * gpg
       * git
       * svn

dev-tools/scripts/smokeTestRelease.py

@@ -22,6 +22,7 @@
 import hashlib
 import http.client
 import os
+import platform
 import re
 import shutil
 import subprocess
@@ -34,8 +35,6 @@
 import xml.etree.ElementTree as ET
 import zipfile
 from collections import namedtuple
-
-import platform
 import scriptutil
 
 # This tool expects to find /solr off the base URL.  You
@@ -142,15 +141,18 @@ def checkJARMetaData(desc, jarFile, gitRevision, version):
 
     s = decodeUTF8(z.read(MANIFEST_FILE_NAME))
 
+    compileJDK = '21'
+    if 'solrj' in desc or 'api' in desc:
+      compileJDK = '17'
     for verify in (
       'Specification-Vendor: The Apache Software Foundation',
       'Implementation-Vendor: The Apache Software Foundation',
       'Specification-Title: Apache Solr Search Server:',
       'Implementation-Title: org.apache.solr',
-      'X-Compile-Source-JDK: 11',
-      'X-Compile-Target-JDK: 11',
+      'X-Compile-Source-JDK: %s' % compileJDK,
+      'X-Compile-Target-JDK: %s' % compileJDK,
       'Specification-Version: %s' % version,
-      'X-Build-JDK: 11.',
+      'X-Build-JDK: 21.',
       'Extension-Name: org.apache.solr'):
       if type(verify) is not tuple:
         verify = (verify,)
@@ -626,8 +628,8 @@ def verifyUnpacked(java, artifact, unpackPath, gitRevision, version, testArgs):
   #     in_root_folder.remove(fileName)
 
   if isSrc:
-    expected_src_root_folders = ['buildSrc', 'dev-docs', 'dev-tools', 'gradle', 'help', 'solr']
-    expected_src_root_files = ['build.gradle', 'gradlew', 'gradlew.bat', 'settings.gradle', 'settings.gradle.lockfile']
+    expected_src_root_folders = ['build-tools', 'dev-docs', 'dev-tools', 'gradle', 'help', 'solr']
+    expected_src_root_files = ['build.gradle', 'gradlew', 'gradlew.bat', 'settings.gradle', 'settings-gradle.lockfile', 'versions.lock']
     expected_src_solr_files = ['build.gradle']
     expected_src_solr_folders = ['benchmark',  'bin', 'modules', 'api', 'core', 'cross-dc-manager', 'docker', 'documentation', 'example', 'licenses', 'packaging', 'distribution', 'prometheus-exporter', 'server', 'solr-ref-guide', 'solrj', 'solrj-streaming', 'solrj-zookeeper', 'test-framework', 'webapp', '.gitignore', '.gitattributes']
     is_in_list(in_root_folder, expected_src_root_folders)
@@ -661,47 +663,47 @@ def verifyUnpacked(java, artifact, unpackPath, gitRevision, version, testArgs):
 
     validateCmd = './gradlew --no-daemon check -p solr/documentation'
     print('    run "%s"' % validateCmd)
-    java.run_java11(validateCmd, '%s/validate.log' % unpackPath)
-
-    print("    run tests w/ Java 11 and testArgs='%s'..." % testArgs)
-    java.run_java11('./gradlew --no-daemon test %s' % testArgs, '%s/test.log' % unpackPath)
-    print("    run integration tests w/ Java 11")
-    java.run_java11('./gradlew --no-daemon integrationTest -Dversion.release=%s' % version, '%s/itest.log' % unpackPath)
-    print("    build binary release w/ Java 11")
-    java.run_java11('./gradlew --no-daemon dev -Dversion.release=%s' % version, '%s/assemble.log' % unpackPath)
-    testSolrExample("%s/solr/packaging/build/dev" % unpackPath, java.java11_home, False)
-
-    if java.run_java17:
-      print("    run tests w/ Java 17 and testArgs='%s'..." % testArgs)
-      java.run_java17('./gradlew --no-daemon clean test %s' % testArgs, '%s/test-java17.log' % unpackPath)
-      print("    run integration tests w/ Java 17")
-      java.run_java17('./gradlew --no-daemon integrationTest -Dversion.release=%s' % version, '%s/itest-java17.log' % unpackPath)
-      print("    build binary release w/ Java 17")
-      java.run_java17('./gradlew --no-daemon dev -Dversion.release=%s' % version, '%s/assemble-java17.log' % unpackPath)
-      testSolrExample("%s/solr/packaging/build/dev" % unpackPath, java.java17_home, False)
+    java.run_java21(validateCmd, '%s/validate.log' % unpackPath)
+
+    print("    run tests w/ Java 21 and testArgs='%s'..." % testArgs)
+    java.run_java21('./gradlew --no-daemon test %s' % testArgs, '%s/test.log' % unpackPath)
+    print("    run integration tests w/ Java 21")
+    java.run_java21('./gradlew --no-daemon integrationTest -Dversion.release=%s' % version, '%s/itest.log' % unpackPath)
+    print("    build binary release w/ Java 21")
+    java.run_java21('./gradlew --no-daemon dev -Dversion.release=%s' % version, '%s/assemble.log' % unpackPath)
+    testSolrExample("%s/solr/packaging/build/dev" % unpackPath, java.java21_home, False)
+
+    if java.run_java25:
+      print("    run tests w/ Java 25 and testArgs='%s'..." % testArgs)
+      java.run_java25('./gradlew --no-daemon clean test %s' % testArgs, '%s/test-java25.log' % unpackPath)
+      print("    run integration tests w/ Java 25")
+      java.run_java25('./gradlew --no-daemon integrationTest -Dversion.release=%s' % version, '%s/itest-java25.log' % unpackPath)
+      print("    build binary release w/ Java 25")
+      java.run_java25('./gradlew --no-daemon dev -Dversion.release=%s' % version, '%s/assemble-java25.log' % unpackPath)
+      testSolrExample("%s/solr/packaging/build/dev" % unpackPath, java.java25_home, False)
 
   else:
     # Binary tarball
     checkAllJARs(os.getcwd(), gitRevision, version)
 
-    print('    copying unpacked distribution for Java 11 ...')
-    java11UnpackPath = '%s-java11' % unpackPath
-    if os.path.exists(java11UnpackPath):
-      shutil.rmtree(java11UnpackPath)
-    shutil.copytree(unpackPath, java11UnpackPath)
-    os.chdir(java11UnpackPath)
-    print('    test solr example w/ Java 11...')
-    testSolrExample(java11UnpackPath, java.java11_home, isSlim)
-
-    if java.run_java17:
-      print('    copying unpacked distribution for Java 17 ...')
-      java17UnpackPath = '%s-java17' % unpackPath
-      if os.path.exists(java17UnpackPath):
-        shutil.rmtree(java17UnpackPath)
-      shutil.copytree(unpackPath, java17UnpackPath)
-      os.chdir(java17UnpackPath)
-      print('    test solr example w/ Java 17...')
-      testSolrExample(java17UnpackPath, java.java17_home, isSlim)
+    print('    copying unpacked distribution for Java 21 ...')
+    java21UnpackPath = '%s-java21' % unpackPath
+    if os.path.exists(java21UnpackPath):
+      shutil.rmtree(java21UnpackPath)
+    shutil.copytree(unpackPath, java21UnpackPath)
+    os.chdir(java21UnpackPath)
+    print('    test solr example w/ Java 21...')
+    testSolrExample(java21UnpackPath, java.java21_home, isSlim)
+
+    if java.run_java25:
+      print('    copying unpacked distribution for Java 25 ...')
+      java25UnpackPath = '%s-java25' % unpackPath
+      if os.path.exists(java25UnpackPath):
+        shutil.rmtree(java25UnpackPath)
+      shutil.copytree(unpackPath, java25UnpackPath)
+      os.chdir(java25UnpackPath)
+      print('    test solr example w/ Java 25...')
+      testSolrExample(java25UnpackPath, java.java25_home, isSlim)
 
     os.chdir(unpackPath)
 
@@ -1020,7 +1022,7 @@ def crawl(downloadedFiles, urlString, targetDir, exclusions=set()):
         sys.stdout.write('.')
 
 
-def make_java_config(parser, java17_home):
+def make_java_config(parser, java25_home):
   def _make_runner(java_home, version):
     print('Java %s JAVA_HOME=%s' % (version, java_home))
     if cygwin:
@@ -1034,23 +1036,23 @@ def _make_runner(java_home, version):
     def run_java(cmd, logfile):
       run('%s; %s' % (cmd_prefix, cmd), logfile)
     return run_java
-  java11_home =  os.environ.get('JAVA_HOME')
-  if java11_home is None:
+  java21_home =  os.environ.get('JAVA_HOME')
+  if java21_home is None:
     parser.error('JAVA_HOME must be set')
-  run_java11 = _make_runner(java11_home, '11')
-  run_java17 = None
-  if java17_home is not None:
-    run_java17 = _make_runner(java17_home, '17')
+  run_java21 = _make_runner(java21_home, '21')
+  run_java25 = None
+  if java25_home is not None:
+    run_java25 = _make_runner(java25_home, '25')
 
-  jc = namedtuple('JavaConfig', 'run_java11 java11_home run_java17 java17_home')
-  return jc(run_java11, java11_home, run_java17, java17_home)
+  jc = namedtuple('JavaConfig', 'run_java21 java21_home run_java25 java25_home')
+  return jc(run_java21, java21_home, run_java25, java25_home)
 
 version_re = re.compile(r'(\d+\.\d+\.\d+(-ALPHA|-BETA)?)')
 revision_re = re.compile(r'rev-([a-f\d]+)')
 def parse_config():
   epilogue = textwrap.dedent('''
     Example usage:
-    python3 -u dev-tools/scripts/smokeTestRelease.py https://dist.apache.org/repos/dist/dev/solr/solr-9.0.0-RC1-rev-c7510a0...
+    python3 -u dev-tools/scripts/smokeTestRelease.py https://dist.apache.org/repos/dist/dev/solr/solr-10.0.0-RC1-rev-c7510a0...
   ''')
   description = 'Utility to test a release.'
   parser = argparse.ArgumentParser(description=description, epilog=epilogue,
@@ -1065,8 +1067,8 @@ def parse_config():
                       help='GIT revision number that release was built with, defaults to that in URL')
   parser.add_argument('--version', metavar='X.Y.Z(-ALPHA|-BETA)?',
                       help='Version of the release, defaults to that in URL')
-  parser.add_argument('--test-java17', metavar='java17_home',
-                      help='Path to Java17 home directory, to run tests with if specified')
+  parser.add_argument('--test-java25', metavar='java25_home',
+                      help='Path to Java25 home directory, to run tests with if specified')
   parser.add_argument('--download-only', action='store_true', default=False,
                       help='Only perform download and sha hash check steps')
   parser.add_argument('--dev-mode', action='store_true', default=False,
@@ -1095,7 +1097,7 @@ def parse_config():
   if c.local_keys is not None and not os.path.exists(c.local_keys):
     parser.error('Local KEYS file "%s" not found' % c.local_keys)
 
-  c.java = make_java_config(parser, c.test_java17)
+  c.java = make_java_config(parser, c.test_java25)
 
   if c.tmp_dir:
     c.tmp_dir = os.path.abspath(c.tmp_dir)