SCANJLIB-235 Set deprecated SSL and proxy properties for SQ < 10.6 (#203

)

* SCANJLIB-235 Extract Http properties parsing in a separate class
* Rename ServerConnection -> ScannerHttpClient
* Move userAgent to HttpConfig
* Move the other properties to HttpConfig
* SCANJLIB-235 Set deprecated properties for SQ < 10.6

its/it-tests/src/test/java/com/sonar/scanner/lib/it/SSLTest.java

@@ -115,7 +115,7 @@ private static void startSSLTransparentReverseProxy(boolean requireClientAuth) t
 
     // Handler Structure
     HandlerCollection handlers = new HandlerCollection();
-    handlers.setHandlers(new Handler[]{proxyHandler(), new DefaultHandler()});
+    handlers.setHandlers(new Handler[] {proxyHandler(), new DefaultHandler()});
     server.setHandler(handlers);
 
     ServerConnector http = new ServerConnector(server, new HttpConnectionFactory(httpConfig));
@@ -227,8 +227,7 @@ public void simple_analysis_with_server_and_without_client_certificate_is_failin
         p.matches(failedAnalysis + "Caused by: javax\\.net\\.ssl\\.SSLProtocolException: Broken pipe \\(Write failed\\).*") ||
         p.matches(failedAnalysis + "Caused by: javax\\.net\\.ssl\\.SSLHandshakeException: Received fatal alert: bad_certificate.*") ||
         p.matches(failedAnalysis + "Caused by: java\\.net\\.SocketException: Broken pipe.*") ||
-        p.matches(failedAnalysis + "Caused by: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target.*")
-      );
+        p.matches(failedAnalysis + "Caused by: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target.*"));
   }
 
   private static Path project(String projectName) {
@@ -238,7 +237,8 @@ private static Path project(String projectName) {
   @Test
   @UseDataProvider("variousClientTrustStores")
   public void simple_analysis_with_server_certificate(String clientTrustStore, String keyStorePassword, boolean useJavaSslProperties) throws Exception {
-    assumeTrue("New SSL properties have been introduced in 10.6", ORCHESTRATOR.getServer().version().isGreaterThanOrEquals(10,6) || useJavaSslProperties);
+    assumeTrue("Support for PKCS12 keystore generated by openssl was added in SQ 10.7", !clientTrustStore.endsWith("-openssl.p12") ||
+      ORCHESTRATOR.getServer().version().isGreaterThanOrEquals(10, 7));
     startSSLTransparentReverseProxy(false);
     SimpleScanner scanner = new SimpleScanner();
 

lib/src/main/java/org/sonarsource/scanner/lib/ScannerEngineBootstrapper.java

@@ -19,14 +19,18 @@
  */
 package org.sonarsource.scanner.lib;
 
+import java.net.InetSocketAddress;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.time.temporal.ChronoUnit;
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.sonarsource.scanner.lib.internal.ArchResolver;
@@ -36,12 +40,12 @@
 import org.sonarsource.scanner.lib.internal.Paths2;
 import org.sonarsource.scanner.lib.internal.ScannerEngineLauncherFactory;
 import org.sonarsource.scanner.lib.internal.cache.FileCache;
-import org.sonarsource.scanner.lib.internal.http.ServerConnection;
+import org.sonarsource.scanner.lib.internal.http.HttpConfig;
+import org.sonarsource.scanner.lib.internal.http.ScannerHttpClient;
 import org.sonarsource.scanner.lib.internal.util.VersionUtils;
 
 import static org.sonarsource.scanner.lib.ScannerProperties.SCANNER_ARCH;
 import static org.sonarsource.scanner.lib.ScannerProperties.SCANNER_OS;
-import static org.sonarsource.scanner.lib.internal.http.ServerConnection.removeTrailingSlash;
 
 /**
  * Entry point to run a Sonar analysis programmatically.
@@ -57,14 +61,14 @@ public class ScannerEngineBootstrapper {
   private final IsolatedLauncherFactory launcherFactory;
   private final ScannerEngineLauncherFactory scannerEngineLauncherFactory;
   private final Map<String, String> bootstrapProperties = new HashMap<>();
-  private final ServerConnection serverConnection;
+  private final ScannerHttpClient scannerHttpClient;
   private final System2 system;
 
   ScannerEngineBootstrapper(String app, String version, System2 system,
-    ServerConnection serverConnection, IsolatedLauncherFactory launcherFactory,
+    ScannerHttpClient scannerHttpClient, IsolatedLauncherFactory launcherFactory,
     ScannerEngineLauncherFactory scannerEngineLauncherFactory) {
     this.system = system;
-    this.serverConnection = serverConnection;
+    this.scannerHttpClient = scannerHttpClient;
     this.launcherFactory = launcherFactory;
     this.scannerEngineLauncherFactory = scannerEngineLauncherFactory;
     this.setBootstrapProperty(InternalProperties.SCANNER_APP, app)
@@ -73,7 +77,7 @@ public class ScannerEngineBootstrapper {
 
   public static ScannerEngineBootstrapper create(String app, String version) {
     System2 system = new System2();
-    return new ScannerEngineBootstrapper(app, version, system, new ServerConnection(),
+    return new ScannerEngineBootstrapper(app, version, system, new ScannerHttpClient(),
       new IsolatedLauncherFactory(), new ScannerEngineLauncherFactory(system));
   }
 
@@ -106,20 +110,62 @@ public ScannerEngineFacade bootstrap() {
     var isSimulation = properties.containsKey(InternalProperties.SCANNER_DUMP_TO_FILE);
     var sonarUserHome = resolveSonarUserHome(properties);
     var fileCache = FileCache.create(sonarUserHome);
-    serverConnection.init(properties, sonarUserHome);
+    var httpConfig = new HttpConfig(bootstrapProperties, sonarUserHome);
+    scannerHttpClient.init(httpConfig);
     String serverVersion = null;
     if (!isSonarCloud) {
-      serverVersion = getServerVersion(serverConnection, isSimulation, properties);
+      serverVersion = getServerVersion(scannerHttpClient, isSimulation, properties);
     }
 
     if (isSimulation) {
       return new SimulationScannerEngineFacade(properties, isSonarCloud, serverVersion);
     } else if (isSonarCloud || VersionUtils.isAtLeastIgnoringQualifier(serverVersion, SQ_VERSION_NEW_BOOTSTRAPPING)) {
-      var launcher = scannerEngineLauncherFactory.createLauncher(serverConnection, fileCache, properties);
+      var launcher = scannerEngineLauncherFactory.createLauncher(scannerHttpClient, fileCache, properties);
       return new NewScannerEngineFacade(properties, launcher, isSonarCloud, serverVersion);
     } else {
-      var launcher = launcherFactory.createLauncher(serverConnection, fileCache);
-      return new InProcessScannerEngineFacade(properties, launcher, false, serverVersion);
+      var launcher = launcherFactory.createLauncher(scannerHttpClient, fileCache);
+      var adaptedProperties = adaptDeprecatedProperties(properties, httpConfig);
+      return new InProcessScannerEngineFacade(adaptedProperties, launcher, false, serverVersion);
+    }
+  }
+
+  /**
+   * Older SonarQube versions used to rely on some different properties, or even {@link System} properties.
+   * For backward compatibility, we adapt the new properties to the old format.
+   */
+  @Nonnull
+  Map<String, String> adaptDeprecatedProperties(Map<String, String> properties, HttpConfig httpConfig) {
+    var adaptedProperties = new HashMap<>(properties);
+    if (!adaptedProperties.containsKey(HttpConfig.READ_TIMEOUT_SEC_PROPERTY)) {
+      adaptedProperties.put(HttpConfig.READ_TIMEOUT_SEC_PROPERTY, "" + httpConfig.getSocketTimeout().get(ChronoUnit.SECONDS));
+    }
+    var proxy = httpConfig.getProxy();
+    if (proxy != null) {
+      setSystemPropertyIfNotAlreadySet("http.proxyHost", ((InetSocketAddress) proxy.address()).getHostString());
+      setSystemPropertyIfNotAlreadySet("https.proxyHost", ((InetSocketAddress) proxy.address()).getHostString());
+      setSystemPropertyIfNotAlreadySet("http.proxyPort", "" + ((InetSocketAddress) proxy.address()).getPort());
+      setSystemPropertyIfNotAlreadySet("https.proxyPort", "" + ((InetSocketAddress) proxy.address()).getPort());
+    }
+    setSystemPropertyIfNotAlreadySet("http.proxyUser", httpConfig.getProxyUser());
+    setSystemPropertyIfNotAlreadySet("http.proxyPassword", httpConfig.getProxyPassword());
+
+    var keyStore = httpConfig.getSslConfig().getKeyStore();
+    if (keyStore != null) {
+      setSystemPropertyIfNotAlreadySet("javax.net.ssl.keyStore", keyStore.getPath().toString());
+      setSystemPropertyIfNotAlreadySet("javax.net.ssl.keyStorePassword", keyStore.getKeyStorePassword());
+    }
+    var trustStore = httpConfig.getSslConfig().getTrustStore();
+    if (trustStore != null) {
+      setSystemPropertyIfNotAlreadySet("javax.net.ssl.trustStore", trustStore.getPath().toString());
+      setSystemPropertyIfNotAlreadySet("javax.net.ssl.trustStorePassword", trustStore.getKeyStorePassword());
+    }
+
+    return Map.copyOf(adaptedProperties);
+  }
+
+  private void setSystemPropertyIfNotAlreadySet(String key, String value) {
+    if (system.getProperty(key) == null && StringUtils.isNotBlank(value)) {
+      System.setProperty(key, value);
     }
   }
 
@@ -134,16 +180,16 @@ private static Path resolveSonarUserHome(Map<String, String> properties) {
     return Paths.get(sonarUserHome);
   }
 
-  private static String getServerVersion(ServerConnection serverConnection, boolean isSimulation, Map<String, String> properties) {
+  private static String getServerVersion(ScannerHttpClient scannerHttpClient, boolean isSimulation, Map<String, String> properties) {
     if (isSimulation) {
       return properties.getOrDefault(InternalProperties.SCANNER_VERSION_SIMULATION, "5.6");
     }
 
     try {
-      return serverConnection.callRestApi("/analysis/version");
+      return scannerHttpClient.callRestApi("/analysis/version");
     } catch (Exception e) {
       try {
-        return serverConnection.callWebApi("/api/server/version");
+        return scannerHttpClient.callWebApi("/api/server/version");
       } catch (Exception e2) {
         var ex = new IllegalStateException("Failed to get server version", e2);
         ex.addSuppressed(e);
@@ -154,8 +200,8 @@ private static String getServerVersion(ServerConnection serverConnection, boolea
 
   private void initBootstrapDefaultValues() {
     setBootstrapPropertyIfNotAlreadySet(ScannerProperties.HOST_URL, getSonarCloudUrl());
-    setBootstrapPropertyIfNotAlreadySet(ScannerProperties.API_BASE_URL, isSonarCloud(bootstrapProperties) ?
-      SONARCLOUD_REST_API : (removeTrailingSlash(bootstrapProperties.get(ScannerProperties.HOST_URL)) + "/api/v2"));
+    setBootstrapPropertyIfNotAlreadySet(ScannerProperties.API_BASE_URL,
+      isSonarCloud(bootstrapProperties) ? SONARCLOUD_REST_API : (StringUtils.removeEnd(bootstrapProperties.get(ScannerProperties.HOST_URL), "/") + "/api/v2"));
     if (!bootstrapProperties.containsKey(SCANNER_OS)) {
       setBootstrapProperty(SCANNER_OS, new OsResolver(system, new Paths2()).getOs().name().toLowerCase(Locale.ENGLISH));
     }

lib/src/main/java/org/sonarsource/scanner/lib/internal/BootstrapIndexDownloader.java

@@ -23,15 +23,15 @@
 import java.util.Collection;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import org.sonarsource.scanner.lib.internal.http.ServerConnection;
+import org.sonarsource.scanner.lib.internal.http.ScannerHttpClient;
 
 class BootstrapIndexDownloader {
 
   private static final Logger LOG = LoggerFactory.getLogger(BootstrapIndexDownloader.class);
 
-  private final ServerConnection conn;
+  private final ScannerHttpClient conn;
 
-  BootstrapIndexDownloader(ServerConnection conn) {
+  BootstrapIndexDownloader(ScannerHttpClient conn) {
     this.conn = conn;
   }
 

lib/src/main/java/org/sonarsource/scanner/lib/internal/IsolatedLauncherFactory.java

@@ -32,7 +32,7 @@
 import org.sonarsource.scanner.lib.internal.batch.IsolatedLauncher;
 import org.sonarsource.scanner.lib.internal.cache.CachedFile;
 import org.sonarsource.scanner.lib.internal.cache.FileCache;
-import org.sonarsource.scanner.lib.internal.http.ServerConnection;
+import org.sonarsource.scanner.lib.internal.http.ScannerHttpClient;
 
 public class IsolatedLauncherFactory {
 
@@ -61,11 +61,11 @@ private IsolatedClassloader createClassLoader(List<Path> jarFiles, ClassloadRule
     return classloader;
   }
 
-  public IsolatedLauncherAndClassloader createLauncher(ServerConnection serverConnection, FileCache fileCache) {
+  public IsolatedLauncherAndClassloader createLauncher(ScannerHttpClient scannerHttpClient, FileCache fileCache) {
     Set<String> unmaskRules = new HashSet<>();
     unmaskRules.add("org.sonarsource.scanner.lib.internal.batch.");
     ClassloadRules rules = new ClassloadRules(Collections.emptySet(), unmaskRules);
-    LegacyScannerEngineDownloader legacyScannerEngineDownloader = new LegacyScannerEngineDownloaderFactory(serverConnection, fileCache).create();
+    LegacyScannerEngineDownloader legacyScannerEngineDownloader = new LegacyScannerEngineDownloaderFactory(scannerHttpClient, fileCache).create();
     return createLauncher(legacyScannerEngineDownloader, rules);
   }
 

lib/src/main/java/org/sonarsource/scanner/lib/internal/JavaRunnerFactory.java

@@ -46,7 +46,7 @@
 import org.sonarsource.scanner.lib.internal.cache.CachedFile;
 import org.sonarsource.scanner.lib.internal.cache.FileCache;
 import org.sonarsource.scanner.lib.internal.cache.HashMismatchException;
-import org.sonarsource.scanner.lib.internal.http.ServerConnection;
+import org.sonarsource.scanner.lib.internal.http.ScannerHttpClient;
 import org.sonarsource.scanner.lib.internal.util.CompressionUtils;
 
 import static java.lang.String.format;
@@ -72,7 +72,7 @@ public JavaRunnerFactory(System2 system, ProcessWrapperFactory processWrapperFac
     this.processWrapperFactory = processWrapperFactory;
   }
 
-  public JavaRunner createRunner(ServerConnection serverConnection, FileCache fileCache, Map<String, String> properties) {
+  public JavaRunner createRunner(ScannerHttpClient scannerHttpClient, FileCache fileCache, Map<String, String> properties) {
     String javaExecutablePropValue = properties.get(JAVA_EXECUTABLE_PATH);
     if (javaExecutablePropValue != null) {
       LOG.info("Using the configured java executable '{}'", javaExecutablePropValue);
@@ -82,7 +82,7 @@ public JavaRunner createRunner(ServerConnection serverConnection, FileCache file
     if (skipJreProvisioning) {
       LOG.info("JRE provisioning is disabled");
     } else {
-      var cachedFile = getJreFromServer(serverConnection, fileCache, properties, true);
+      var cachedFile = getJreFromServer(scannerHttpClient, fileCache, properties, true);
       if (cachedFile.isPresent()) {
         return new JavaRunner(cachedFile.get().getPathInCache(), cachedFile.get().isCacheHit() ? JreCacheHit.HIT : JreCacheHit.MISS);
       }
@@ -129,34 +129,34 @@ private Path findJavaInPath(String javaExe) {
     }
   }
 
-  private static Optional<CachedFile> getJreFromServer(ServerConnection serverConnection, FileCache fileCache, Map<String, String> properties, boolean retry) {
+  private static Optional<CachedFile> getJreFromServer(ScannerHttpClient scannerHttpClient, FileCache fileCache, Map<String, String> properties, boolean retry) {
     String os = properties.get(SCANNER_OS);
     String arch = properties.get(SCANNER_ARCH);
     LOG.info("JRE provisioning: os[{}], arch[{}]", os, arch);
 
     try {
-      var jreMetadata = getJreMetadata(serverConnection, os, arch);
+      var jreMetadata = getJreMetadata(scannerHttpClient, os, arch);
       if (jreMetadata.isEmpty()) {
         LOG.info("No JRE found for this OS/architecture");
         return Optional.empty();
       }
       var cachedFile = fileCache.getOrDownload(jreMetadata.get().getFilename(), jreMetadata.get().getSha256(), "SHA-256",
-        new JreDownloader(serverConnection, jreMetadata.get()));
+        new JreDownloader(scannerHttpClient, jreMetadata.get()));
       var extractedDirectory = extractArchive(cachedFile.getPathInCache());
       return Optional.of(new CachedFile(extractedDirectory.resolve(jreMetadata.get().javaPath), cachedFile.isCacheHit()));
     } catch (HashMismatchException e) {
       if (retry) {
         // A new JRE might have been published between the metadata fetch and the download
         LOG.warn("Failed to get the JRE, retrying...");
-        return getJreFromServer(serverConnection, fileCache, properties, false);
+        return getJreFromServer(scannerHttpClient, fileCache, properties, false);
       }
       throw e;
     }
   }
 
-  private static Optional<JreMetadata> getJreMetadata(ServerConnection serverConnection, String os, String arch) {
+  private static Optional<JreMetadata> getJreMetadata(ScannerHttpClient scannerHttpClient, String os, String arch) {
     try {
-      String response = serverConnection.callRestApi(format(API_PATH_JRE + "?os=%s&arch=%s", os, arch));
+      String response = scannerHttpClient.callRestApi(format(API_PATH_JRE + "?os=%s&arch=%s", os, arch));
       Type listType = new TypeToken<ArrayList<JreMetadata>>() {
       }.getType();
       List<JreMetadata> jres = new Gson().fromJson(response, listType);
@@ -239,10 +239,10 @@ private static void extract(Path compressedFile, Path targetDir) throws IOExcept
   }
 
   static class JreDownloader implements FileCache.Downloader {
-    private final ServerConnection connection;
+    private final ScannerHttpClient connection;
     private final JreMetadata jreMetadata;
 
-    JreDownloader(ServerConnection connection, JreMetadata jreMetadata) {
+    JreDownloader(ScannerHttpClient connection, JreMetadata jreMetadata) {
       this.connection = connection;
       this.jreMetadata = jreMetadata;
     }

lib/src/main/java/org/sonarsource/scanner/lib/internal/LegacyScannerEngineDownloader.java

@@ -30,7 +30,7 @@
 import org.sonarsource.scanner.lib.internal.BootstrapIndexDownloader.JarEntry;
 import org.sonarsource.scanner.lib.internal.cache.CachedFile;
 import org.sonarsource.scanner.lib.internal.cache.FileCache;
-import org.sonarsource.scanner.lib.internal.http.ServerConnection;
+import org.sonarsource.scanner.lib.internal.http.ScannerHttpClient;
 
 import static java.lang.String.format;
 
@@ -70,9 +70,9 @@ private List<CachedFile> getOrDownloadScannerEngineFiles() {
   }
 
   static class ScannerFileDownloader implements FileCache.Downloader {
-    private final ServerConnection connection;
+    private final ScannerHttpClient connection;
 
-    ScannerFileDownloader(ServerConnection conn) {
+    ScannerFileDownloader(ScannerHttpClient conn) {
       this.connection = conn;
     }
 

lib/src/main/java/org/sonarsource/scanner/lib/internal/LegacyScannerEngineDownloaderFactory.java

@@ -20,20 +20,20 @@
 package org.sonarsource.scanner.lib.internal;
 
 import org.sonarsource.scanner.lib.internal.cache.FileCache;
-import org.sonarsource.scanner.lib.internal.http.ServerConnection;
+import org.sonarsource.scanner.lib.internal.http.ScannerHttpClient;
 
 class LegacyScannerEngineDownloaderFactory {
-  private final ServerConnection serverConnection;
+  private final ScannerHttpClient scannerHttpClient;
   private final FileCache fileCache;
 
-  LegacyScannerEngineDownloaderFactory(ServerConnection conn, FileCache fileCache) {
-    this.serverConnection = conn;
+  LegacyScannerEngineDownloaderFactory(ScannerHttpClient conn, FileCache fileCache) {
+    this.scannerHttpClient = conn;
     this.fileCache = fileCache;
   }
 
   LegacyScannerEngineDownloader create() {
-    BootstrapIndexDownloader bootstrapIndexDownloader = new BootstrapIndexDownloader(serverConnection);
-    LegacyScannerEngineDownloader.ScannerFileDownloader scannerFileDownloader = new LegacyScannerEngineDownloader.ScannerFileDownloader(serverConnection);
+    BootstrapIndexDownloader bootstrapIndexDownloader = new BootstrapIndexDownloader(scannerHttpClient);
+    LegacyScannerEngineDownloader.ScannerFileDownloader scannerFileDownloader = new LegacyScannerEngineDownloader.ScannerFileDownloader(scannerHttpClient);
     JarExtractor jarExtractor = new JarExtractor();
     return new LegacyScannerEngineDownloader(scannerFileDownloader, bootstrapIndexDownloader, fileCache, jarExtractor);
   }