Update to latest master

CMakeInstallation.cmake

@@ -74,7 +74,7 @@ FIND_PROGRAM(NC_DPKG NAMES dpkg)
 IF(NC_DPKG)
   # Define a macro for getting the dpkg architecture.
   MACRO(getdpkg_arch arch)
-    exec_program("${NC_DPKG}" ARGS "--print-architecture" OUTPUT_VARIABLE "${arch}")
+    execute_process(COMMAND "${NC_DPKG}" "--print-architecture" OUTPUT_VARIABLE "${arch}" OUTPUT_STRIP_TRAILING_WHITESPACE)
   ENDMACRO(getdpkg_arch)
   getdpkg_arch(dpkg_arch)
 

CMakeLists.txt

@@ -6,9 +6,7 @@
 # Set Project Properties
 ##################################
 
-#Minimum required CMake Version
 cmake_minimum_required(VERSION 3.12.0)
-# CMake 3.12: Use libraries specified in CMAKE_REQUIRED_LIBRARIES for check include macros
 
 #Project Name
 project(netCDF
@@ -18,6 +16,14 @@ DESCRIPTION "NetCDF is a set of software libraries and machine-independent data
 )
 set(PACKAGE "netCDF" CACHE STRING "")
 
+# Backport of built-in `PROJECT_IS_TOP_LEVEL` from CMake 3.21
+if (NOT DEFINED NETCDF_IS_TOP_LEVEL)
+  set(NETCDF_IS_TOP_LEVEL OFF)
+  if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR)
+    set(NETCDF_IS_TOP_LEVEL ON)
+  endif ()
+endif ()
+
 #####
 # Version Info:
 #
@@ -51,7 +57,7 @@ SET(NC_DISPATCH_VERSION 5)
 find_program(UNAME NAMES uname)
 IF(UNAME)
   macro(getuname name flag)
-    exec_program("${UNAME}" ARGS "${flag}" OUTPUT_VARIABLE "${name}")
+    execute_process(COMMAND "${UNAME}" "${flag}" OUTPUT_VARIABLE "${name}" OUTPUT_STRIP_TRAILING_WHITESPACE)
   endmacro(getuname)
   getuname(osname -s)
   getuname(osrel  -r)
@@ -1405,7 +1411,7 @@ IF(ENABLE_TESTS)
     SET(HOSTNAME_ARG "-s")
   ENDIF()
   IF(HOSTNAME_CMD)
-    EXEC_PROGRAM(${HOSTNAME_CMD} ARGS "${HOSTNAME_ARG}" OUTPUT_VARIABLE HOSTNAME)
+    EXECUTE_PROCESS(COMMAND ${HOSTNAME_CMD} "${HOSTNAME_ARG}" OUTPUT_VARIABLE HOSTNAME OUTPUT_STRIP_TRAILING_WHITESPACE)
     SET(NC_CTEST_SITE "${HOSTNAME}" CACHE STRING "Hostname of test machine.")
   ENDIF()
 
@@ -2790,5 +2796,8 @@ install(
 ####
 
 # CPack inclusion must come last.
-# INCLUDE(CPack)
-INCLUDE(CMakeInstallation.cmake)
+option(NETCDF_PACKAGE "Create netCDF-C package " ${NETCDF_IS_TOP_LEVEL})
+
+if (NETCDF_PACKAGE)
+  include(CMakeInstallation.cmake)
+endif()

Makefile.am

@@ -211,7 +211,11 @@ install-data-hook:
 all-local: liblib/libnetcdf.la
 	echo ${PACKAGE_VERSION} > VERSION
 if ENABLE_S3_TESTALL
+	rm -f ${abs_top_builddir}/tmp_@[email protected]
 	echo "@TESTUID@" >> ${abs_top_builddir}/s3cleanup_@[email protected]
+	cat ${abs_top_builddir}/s3cleanup_@[email protected] | sort | uniq > ${abs_top_builddir}/tmp_@[email protected]
+	rm -f ${abs_top_builddir}/s3cleanup_@[email protected]
+	mv ${abs_top_builddir}/tmp_@[email protected] ${abs_top_builddir}/s3cleanup_@[email protected]
 endif
 
 if ENABLE_S3_TESTALL

RELEASE_NOTES.md

@@ -8,7 +8,9 @@ This file contains a high-level description of this package's evolution. Release
 ## 4.9.3 - TBD
 
 * Obviate a number of irrelevant warnings. See [Github #2781](https://github.com/Unidata/netcdf-c/pull/2781).
-* Mitigate the problem of remote/nczarr-related test interference. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755).
+* Fix DAP2 proxy problems. See [Github #2764](https://github.com/Unidata/netcdf-c/pull/2764).
+* Cleanup a number of misc issues. See [Github #2763](https://github.com/Unidata/netcdf-c/pull/2763).
+* Mitigate the problem of test interference. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755).
 * Extend NCZarr to support unlimited dimensions. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755).
 * Fix significant bug in the NCZarr cache management. See [Github #2737](https://github.com/Unidata/netcdf-c/pull/2737).
 * Fix default parameters for caching of NCZarr. See [Github #2734](https://github.com/Unidata/netcdf-c/pull/2734).

include/ncrc.h

@@ -53,6 +53,9 @@ typedef struct NCRCinfo {
 	NClist* s3profiles; /* NClist<struct AWSprofile*> */
 } NCRCinfo;
 
+/* Opaque structures */
+struct NCS3INFO;
+
 #if defined(__cplusplus)
 extern "C" {
 #endif
@@ -94,7 +97,7 @@ EXTERNL int NC_getactives3profile(NCURI* uri, const char** profilep);
 EXTERNL int NC_s3profilelookup(const char* profile, const char* key, const char** valuep);
 EXTERNL int NC_authgets3profile(const char* profile, struct AWSprofile** profilep);
 EXTERNL int NC_iss3(NCURI* uri);
-EXTERNL int NC_s3urlrebuild(NCURI* url, char** inoutbucketp, char** inoutregionp, NCURI** newurlp);
+EXTERNL int NC_s3urlrebuild(NCURI* url, struct NCS3INFO* s3, NCURI** newurlp);
 
 #if defined(__cplusplus)
 }

include/ncs3sdk.h

@@ -6,13 +6,19 @@
 #ifndef NCS3SDK_H
 #define NCS3SDK_H 1
 
+/* Track the server type, if known */
+typedef enum NCS3SVC {NCS3UNK=0, /* unknown */
+                 NCS3=1,    /* s3.amazon.aws */
+		 NCS3GS=0   /* storage.googleapis.com */
+} NCS3SVC;
 
 typedef struct NCS3INFO {
     char* host; /* non-null if other*/
     char* region; /* region */
     char* bucket; /* bucket name */
     char* rootkey;
     char* profile;
+    NCS3SVC svc;
 } NCS3INFO;
 
 #ifdef __cplusplus

libdap4/d4curlfunctions.c

@@ -3,6 +3,11 @@
  *   See netcdf/COPYRIGHT file for copying and redistribution conditions.
  *********************************************************************/
 
+/* WARNING: oc2/occurlfunctions.c and libdap4/d4curlfunctions.c
+should be merged since they are essentially the same file.
+In the meantime, changes to one should be propagated to the other.
+*/
+
 #include "d4includes.h"
 #include "d4curlfunctions.h"
 
@@ -123,33 +128,43 @@ set_curlflag(NCD4INFO* state, int flag)
 	    }
 	}
 	break;
-    case CURLOPT_USE_SSL:
-    case CURLOPT_SSLCERT: case CURLOPT_SSLKEY:
-    case CURLOPT_SSL_VERIFYPEER: case CURLOPT_SSL_VERIFYHOST:
-    {
-        struct ssl* ssl = &state->auth->ssl;
+    case CURLOPT_SSL_VERIFYPEER:
 	/* VERIFYPEER == 0 => VERIFYHOST == 0 */
 	/* We need to have 2 states: default and a set value */
-	/* So -1 => default, >= 0 => use value; */
-	if(ssl->verifypeer >= 0)
-            SETCURLOPT(state, CURLOPT_SSL_VERIFYPEER, (OPTARG)(ssl->verifypeer));
+	/* So -1 => default >= 0 => use value */
+	if(state->auth->ssl.verifypeer >= 0) {
+            SETCURLOPT(state, CURLOPT_SSL_VERIFYPEER, (OPTARG)(state->auth->ssl.verifypeer));
+	    if(state->auth->ssl.verifypeer == 0) state->auth->ssl.verifyhost = 0;
+        }
+	break;
+    case CURLOPT_SSL_VERIFYHOST:
 #ifdef HAVE_LIBCURL_766
-	if(ssl->verifyhost >= 0)
-            SETCURLOPT(state, CURLOPT_SSL_VERIFYHOST, (OPTARG)(ssl->verifyhost));
+	if(state->auth->ssl.verifyhost >= 0) {
+            SETCURLOPT(state, CURLOPT_SSL_VERIFYHOST, (OPTARG)(state->auth->ssl.verifyhost));
+	}
 #endif
-        if(ssl->certificate)
-            SETCURLOPT(state, CURLOPT_SSLCERT, ssl->certificate);
-        if(ssl->key)
-            SETCURLOPT(state, CURLOPT_SSLKEY, ssl->key);
-        if(ssl->keypasswd)
+	break;
+    case CURLOPT_SSLCERT:
+        if(state->auth->ssl.certificate)
+            SETCURLOPT(state, CURLOPT_SSLCERT, state->auth->ssl.certificate);
+	break;
+    case CURLOPT_SSLKEY:
+        if(state->auth->ssl.key)
+            SETCURLOPT(state, CURLOPT_SSLKEY, state->auth->ssl.key);
+        if(state->auth->ssl.keypasswd)
             /* libcurl prior to 7.16.4 used 'CURLOPT_SSLKEYPASSWD' */
-            SETCURLOPT(state, CURLOPT_KEYPASSWD, ssl->keypasswd);
-        if(ssl->cainfo)
-            SETCURLOPT(state, CURLOPT_CAINFO, ssl->cainfo);
-        if(ssl->capath)
-            SETCURLOPT(state, CURLOPT_CAPATH, ssl->capath);
-    }
-    break;
+            SETCURLOPT(state, CURLOPT_SSLKEYPASSWD, state->auth->ssl.keypasswd);
+	break;
+    case CURLOPT_CAINFO:
+        if(state->auth->ssl.cainfo)
+            SETCURLOPT(state, CURLOPT_CAINFO, state->auth->ssl.cainfo);
+	break;
+    case CURLOPT_CAPATH:
+	if(state->auth->ssl.capath)
+            SETCURLOPT(state, CURLOPT_CAPATH, state->auth->ssl.capath);
+        break;
+    case CURLOPT_USE_SSL:
+	break;
 
 #ifdef HAVE_CURLOPT_BUFFERSIZE
     case CURLOPT_BUFFERSIZE:
@@ -200,6 +215,12 @@ NCD4_set_flags_perlink(NCD4INFO* state)
     if(ret == NC_NOERR) ret = set_curlflag(state,CURLOPT_COOKIEJAR);
     if(ret == NC_NOERR) ret = set_curlflag(state,CURLOPT_USERPWD);
     if(ret == NC_NOERR) ret = set_curlflag(state,CURLOPT_PROXY);
+    if(ret == NC_NOERR) ret = set_curlflag(state,CURLOPT_SSL_VERIFYPEER);
+    if(ret == NC_NOERR) ret = set_curlflag(state,CURLOPT_SSL_VERIFYHOST);
+    if(ret == NC_NOERR) ret = set_curlflag(state,CURLOPT_SSLCERT);
+    if(ret == NC_NOERR) ret = set_curlflag(state,CURLOPT_SSLKEY);
+    if(ret == NC_NOERR) ret = set_curlflag(state,CURLOPT_CAINFO);
+    if(ret == NC_NOERR) ret = set_curlflag(state,CURLOPT_CAPATH);
     if(ret == NC_NOERR) ret = set_curlflag(state,CURLOPT_USE_SSL);
     if(ret == NC_NOERR) ret = set_curlflag(state, CURLOPT_FOLLOWLOCATION);
     if(ret == NC_NOERR) ret = set_curlflag(state, CURLOPT_MAXREDIRS);

libdispatch/dinfermodel.c

@@ -136,6 +136,7 @@ static const struct MACRODEF {
 {"xarray","mode",{"zarr", NULL}},
 {"noxarray","mode",{"nczarr", "noxarray", NULL}},
 {"zarr","mode",{"nczarr","zarr", NULL}},
+{"gs3","mode",{"gs3","nczarr",NULL}}, /* Google S3 API */
 {NULL,NULL,{NULL}}
 };
 
@@ -196,6 +197,7 @@ static struct NCPROTOCOLLIST {
     {"dods","http","mode=dap2"},
     {"dap4","http","mode=dap4"},
     {"s3","s3","mode=s3"},
+    {"gs3","gs3","mode=gs3"},
     {NULL,NULL,NULL} /* Terminate search */
 };
 
@@ -914,7 +916,7 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void
 	/* If s3, then rebuild the url */
 	if(NC_iss3(uri)) {
 	    NCURI* newuri = NULL;
-	    if((stat = NC_s3urlrebuild(uri,NULL,NULL,&newuri))) goto done;
+	    if((stat = NC_s3urlrebuild(uri,NULL,&newuri))) goto done;
 	    ncurifree(uri);
 	    uri = newuri;
 	} else if(strcmp(uri->protocol,"file")==0) {

libdispatch/drc.c

@@ -437,7 +437,9 @@ rccompile(const char* filepath)
     NCURI* uri = NULL;
     char* nextline = NULL;
     NCglobalstate* globalstate = NC_getglobalstate();
-    char* bucket = NULL;
+    NCS3INFO s3;
+
+    memset(&s3,0,sizeof(s3));
 
     if((ret=NC_readfile(filepath,tmp))) {
         nclog(NCLOGWARN, "Could not open configuration file: %s",filepath);
@@ -484,9 +486,8 @@ rccompile(const char* filepath)
 	    if(NC_iss3(uri)) {
 	         NCURI* newuri = NULL;
 	        /* Rebuild the url to S3 "path" format */
-	        nullfree(bucket);
-		bucket = NULL;
-	        if((ret = NC_s3urlrebuild(uri,&bucket,NULL,&newuri))) goto done;
+		NC_s3clear(&s3);
+	        if((ret = NC_s3urlrebuild(uri,&s3,&newuri))) goto done;
 		ncurifree(uri);
 		uri = newuri;
 		newuri = NULL;
@@ -546,6 +547,7 @@ rccompile(const char* filepath)
     rcorder(rc);
 
 done:
+    NC_s3clear(&s3);
     if(contents) free(contents);
     ncurifree(uri);
     ncbytesfree(tmp);