fix custom s3 host handling LDEV-3594

.github/workflows/main.yml

@@ -3,7 +3,9 @@
 
 name: Java CI
 
-on: [push, pull_request,workflow_dispatch]
+on: [push, pull_request, workflow_dispatch]
+
+concurrency: s3_extension
 
 jobs:
   build:
@@ -61,7 +63,10 @@ jobs:
         testLabels: s3
         testAdditional: ${{ github.workspace }}/tests
         S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_ID_TEST }}
-        S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY_TEST }}
+        SS3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY_TEST }}
         S3_CUSTOM_ACCESS_KEY_ID: "minioadmin"
         S3_CUSTOM_SECRET_KEY: "minioadmin"
-        S3_CUSTOM_HOST: "localhost:9000"
+        S3_CUSTOM_HOST: "http://localhost:9000"
+        S3_GOOGLE_ACCESS_KEY_ID: ${{ secrets.S3_GOOGLE_SECRET_KEY }}
+        SS3_GOOGLE_SECRET_KEY: ${{ secrets.S3_GOOGLE_ACCESS_KEY_ID }}
+        S3_GOOGLE_HOST: "storage.googleapis.com"

build.xml

@@ -92,7 +92,7 @@ resource: "[{'class':'${class}','bundleName':'${bundlename}','bundleVersion':'${
   <target name="compile" depends="copy"
         description="compile the source " >
     <!-- Compile ACF-Infinspan source -->
-    <javac srcdir="${temp}" target="1.7" destdir="${build}" debug="true" debuglevel="lines,vars,source">
+    <javac srcdir="${temp}" target="1.8" source="1.8" destdir="${build}" debug="true" debuglevel="lines,vars,source">
       <classpath refid="classpath" />
     </javac>
   </target>

source/java/src/org/lucee/extension/resource/s3/S3.java

@@ -136,7 +136,7 @@ public S3Bucket createDirectory(String bucketName, AccessControlList acl, String
 		}
 		catch (ServiceException se) {
 			throw toS3Exception(se, "could not create the bucket [" + bucketName
-					+ "], please consult the following website to learn about Bucket Restrictions and limitations: https://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html");
+					+ "] at [" + host + "], please consult the following website to learn about Bucket Restrictions and limitations: https://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html");
 		}
 		catch (FactoryConfigurationError fce) {
 			XMLUtil.validateDocumentBuilderFactory();
@@ -314,8 +314,11 @@ public List<S3Info> list(boolean recursive, boolean listPseudoFolder) throws S3E
 	 * list all allements in a specific bucket
 	 * 
 	 * @param bucketName name of the bucket
+	 * @param objectName name of the object
 	 * @param recursive show all objects (recursive==true) or direct kids
 	 * @param listPseudoFolder if recursive false also list the "folder" of objects with sub folders
+	 * @param onlyChildren 
+	 * @param noCache 
 	 * @return
 	 * @throws S3Exception
 	 * 
@@ -423,8 +426,8 @@ private ValidUntilMap<S3Info> _list(String bucketName, String objectName, boolea
 			// not cached
 			ValidUntilMap<S3Info> _list = timeout <= 0 || noCache ? null : objects.get(key);
 			if (_list == null || _list.validUntil < System.currentTimeMillis()) {
+				/*
 				S3Object[] kids = hasObjName ? getS3Service().listObjects(bucketName, nameFile, ",") : getS3Service().listObjects(bucketName);
-
 				long validUntil = System.currentTimeMillis() + timeout;
 				_list = new ValidUntilMap<S3Info>(validUntil);
 				objects.put(key, _list);
@@ -450,13 +453,71 @@ private ValidUntilMap<S3Info> _list(String bucketName, String objectName, boolea
 						exists.put(toKey(bucketName, name), new ParentObject(bucketName, name, null, validUntil));
 					}
 				}
+				*/
+
+				/* ------------------------ chunked, which includes prefixes ------------------------------ */
+				StorageObjectsChunk chunk = hasObjName ? 
+					listObjectsChunkedAll(bucketName, nameFile, ",", -1)
+					: listObjectsChunkedAll(bucketName, null, null, -1);
+				StorageObject[] _objects = chunk == null ? null : chunk.getObjects();
+				S3Info info;
+
+				long validUntil = System.currentTimeMillis() + timeout;
+				int index;
+				_list = new ValidUntilMap<S3Info>(validUntil);
+				objects.put(key, _list);
+
+				// add bucket
+				if (!hasObjName && !onlyChildren) {
+					S3Bucket b = getS3Service().getBucket(bucketName);
+					_list.put("", new S3BucketWrapper(b, validUntil));
+				}
+
+				ArrayList<String> commonPrefixes = new ArrayList();
+				if (chunk != null && chunk.getCommonPrefixes().length > 0){
+					commonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes()));
+					for (String cp: commonPrefixes){
+						info =  new ParentObject(bucketName, cp, null, validUntil);
+						if (!hasObjName || cp.equals(nameFile) || cp.startsWith(nameDir)) _list.put(cp, info);
+						exists.put(toKey(bucketName, cp), info);
+						// add parent pseudo folders
+						while ((index = cp.lastIndexOf('/')) != -1) {
+							cp = cp.substring(0, index);
+							exists.put(toKey(bucketName, cp), new ParentObject(bucketName, cp, null, validUntil));
+						}
+					}
+				}
+				if ( _objects != null ){
+					String name;
+					StorageObjectWrapper tmp;
+					StorageObject stoObj = null;
+					// direct match
+					for (StorageObject kids: _objects) {
+						name = kids.getName();
+						tmp = new StorageObjectWrapper(this, stoObj = kids, bucketName, validUntil);
+
+						if (!hasObjName || name.equals(nameFile) || name.startsWith(nameDir)) _list.put(kids.getKey(), tmp);
+						exists.put(toKey(kids.getBucketName(), name), tmp);
+
+						// add parent pseudo folders
+						while ((index = name.lastIndexOf('/')) != -1) {
+							name = name.substring(0, index);
+							exists.put(toKey(bucketName, name), new ParentObject(bucketName, name, null, validUntil));
+						}
+					}
+				}
 
 			}
 			return _list;
 		}
-		catch (ServiceException se) {
-			throw toS3Exception(se);
+		catch (FactoryConfigurationError fce) {
+			XMLUtil.validateDocumentBuilderFactory();
+			throw fce;
+		}
+		catch (Exception e) {
+			e.printStackTrace();
 		}
+		return null;
 	}
 
 	private String toKey(String bucketName, String objectName) {
@@ -560,41 +621,56 @@ public S3Info get(String bucketName, final String objectName) throws S3Exception
 			long validUntil = System.currentTimeMillis() + timeout;
 			StorageObject[] objects = chunk == null ? null : chunk.getObjects();
 
-			if (objects == null || objects.length == 0) {
-				exists.put(toKey(bucketName, objectName), new NotExisting(bucketName, objectName, null, validUntil)); // we do not return this, we just store it to cache that it
-																														// does
-				return null;
+			if (chunk !=null && chunk.getCommonPrefixes().length > 0){
+				ArrayList<String> commonPrefixes = new ArrayList();
+				int index;
+				commonPrefixes.addAll(Arrays.asList(chunk.getCommonPrefixes()));
+				for (String cp: commonPrefixes){
+					while ((index = cp.lastIndexOf('/')) != -1) {
+						cp = cp.substring(0, index);
+						if (cp.equals(nameFile) || cp.equals(nameDir))
+							exists.put(toKey(bucketName, cp), info = new ParentObject(bucketName, cp, null, validUntil));
+						else
+							exists.put(toKey(bucketName, cp), new ParentObject(bucketName, cp, null, validUntil));
+					}
+				}
 			}
 
-			String targetName;
-			StorageObject stoObj = null;
-			// direct match
-			for (StorageObject so: objects) {
-				targetName = so.getName();
-				if (nameFile.equals(targetName) || nameDir.equals(targetName)) {
-					exists.put(toKey(bucketName, nameFile), info = new StorageObjectWrapper(this, stoObj = so, bucketName, validUntil));
-				}
+			if (info == null && (objects == null || objects.length == 0)) {
+				exists.put(toKey(bucketName, objectName), new NotExisting(bucketName, objectName, null, validUntil)); // we do not return this, we just store it to cache that it				
 			}
 
-			// pseudo directory?
-			if (info == null) {
+			if (objects != null){
+				String targetName;
+				StorageObject stoObj = null;
+				// direct match
 				for (StorageObject so: objects) {
 					targetName = so.getName();
-					if (nameDir.length() < targetName.length() && targetName.startsWith(nameDir)) {
-						exists.put(toKey(bucketName, nameFile), info = new ParentObject(bucketName, nameDir, null, validUntil));
+					if (nameFile.equals(targetName) || nameDir.equals(targetName)) {
+						exists.put(toKey(bucketName, nameFile), info = new StorageObjectWrapper(this, stoObj = so, bucketName, validUntil));
 					}
 				}
-			}
 
-			for (StorageObject obj: objects) {
-				if (stoObj != null && stoObj.equals(obj)) continue;
-				exists.put(toKey(obj.getBucketName(), obj.getName()), new StorageObjectWrapper(this, obj, bucketName, validUntil));
-			}
+				// pseudo directory?
+				if (info == null) {
+					for (StorageObject so: objects) {
+						targetName = so.getName();
+						if (nameDir.length() < targetName.length() && targetName.startsWith(nameDir)) {
+							exists.put(toKey(bucketName, nameFile), info = new ParentObject(bucketName, nameDir, null, validUntil));
+						}
+					}
+				}
+
+				for (StorageObject obj: objects) {
+					if (stoObj != null && stoObj.equals(obj)) continue;
+					exists.put(toKey(obj.getBucketName(), obj.getName()), new StorageObjectWrapper(this, obj, bucketName, validUntil));
+				}
 
-			if (info == null) {
-				exists.put(toKey(bucketName, objectName), new NotExisting(bucketName, objectName, null, validUntil) // we do not return this, we just store it to cache that it does
-																													// not exis
-				);
+				if (info == null) {
+					exists.put(toKey(bucketName, objectName), new NotExisting(bucketName, objectName, null, validUntil) // we do not return this, we just store it to cache that it does
+																														// not exis
+					);
+				}
 			}
 			return info;
 		}
@@ -621,6 +697,19 @@ public StorageObjectsChunk listObjectsChunkedSilent(String bucketName, String ob
 		return null;
 	}
 
+	public StorageObjectsChunk listObjectsChunkedAll(String bucketName, String objectName, String delim, int max) {
+		try {
+			return getS3Service().listObjectsChunked(bucketName, objectName, delim, max, "", true);
+		}
+		catch (FactoryConfigurationError fce) {
+			XMLUtil.validateDocumentBuilderFactory();
+			throw fce;
+		}
+		catch (Exception e) {
+		}
+		return null;
+	}
+
 	public S3BucketInfo get(String bucketName) throws S3Exception {
 		bucketName = improveBucketName(bucketName);
 
@@ -691,9 +780,14 @@ public void delete(String bucketName, String objectName, boolean force) throws S
 			}
 
 			ObjectKeyAndVersion[] keys = toObjectKeyAndVersions(list, null);
-			if (!force
-					&& (keys.length > 1 || (keys.length == 1 && keys[0].getKey().length() > nameDir.length() && keys[0].getKey().substring(nameDir.length()).indexOf('/') != -1))) {
-				throw new S3Exception("can't delete directory " + bucketName + "/" + objectName + ", directory is not empty");
+			if (!force){
+				// TODO not sure of the logic here, why was it checking only for a sub directory?
+				if (keys.length > 1 || (keys.length == 1 
+						&& keys[0].getKey().length() > nameDir.length() 
+						&& keys[0].getKey().substring(nameDir.length()).length() > 0) //indexOf('/') != -1)
+				) {
+					throw new S3Exception("can't delete directory " + bucketName + "/" + objectName + ", directory is not empty");
+				}
 			}
 
 			// clear cache
@@ -1278,9 +1372,27 @@ private S3Service getS3Service() {
 			synchronized (getToken(accessKeyId + ":" + secretAccessKey)) {
 				if (service == null) {
 					final Jets3tProperties props = Jets3tProperties.getInstance(Constants.JETS3T_PROPERTIES_FILENAME);
-					if (host != null && !host.isEmpty() && !host.equalsIgnoreCase(DEFAULT_HOST)) {
-						props.setProperty("s3service.s3-endpoint", host);
-					}
+					props.clearAllProperties();
+					//if (host != null && !host.isEmpty() && !host.equalsIgnoreCase(DEFAULT_HOST)) {
+						int hasPort = host.indexOf(':');
+						if (hasPort == -1) {
+							props.setProperty("s3service.s3-endpoint", host); // no port, no protocol
+						} else {
+							int hasHttp = host.toLowerCase().indexOf("http", 0); // http(s)://localhost:9000 prefix?
+							if (hasHttp == -1){ // no http prefix
+								props.setProperty("s3service.s3-endpoint", host.substring(0, hasPort));
+								props.setProperty("s3service.s3-endpoint-http-port", host.substring(hasPort + 1));
+							} else { // with http(s) prefix
+								String protocol = host.substring(0,host.indexOf(":"));
+								props.setProperty("s3service.https-only", protocol.equalsIgnoreCase("http") ? "false" : "true" );
+								String _host = host.substring(host.lastIndexOf("/") + 1);
+								hasPort = _host.indexOf(':');
+								props.setProperty("s3service.s3-endpoint", _host.substring(0, hasPort));
+								props.setProperty("s3service.s3-endpoint-http-port", _host.substring(hasPort + 1));
+								props.setProperty("s3service.disable-dns-buckets", "true");
+							}
+						}
+					//}
 					service = new RestS3Service(new AWSCredentials(accessKeyId, secretAccessKey), null, null, props);
 				}
 			}

source/java/src/org/lucee/extension/resource/s3/S3Resource.java

@@ -182,7 +182,11 @@ else if (!Util.isEmpty(s3.getMappingName(), true)) {
 			sb.append(s3.getMappingName()).append("@");
 		}
 
-		if (doHost) sb.append(s3.getHost());
+		if (doHost && s3.getCustomCredentials()){ 
+			if (sb.substring(sb.length()-1) != "@")
+				sb.append("@");
+			sb.append(s3.getHost());
+		}
 
 		return sb.toString();
 	}

source/java/src/org/lucee/extension/resource/s3/S3ResourceProvider.java

@@ -167,9 +167,9 @@ public static String loadWithNewPattern(PageContext pc, S3Properties properties,
 		storage.setValue(defaultLocation);
 
 		int atIndex = path.indexOf('@');
-		int slashIndex = path.indexOf('/');
+		int slashIndex = path.indexOf('/', atIndex); // secret keys can contain /
 		if (slashIndex == -1) {
-			slashIndex = path.length();
+			//slashIndex = path.length(); // never used!
 			path += "/";
 		}
 		int index;
@@ -215,23 +215,35 @@ public static String loadWithNewPattern(PageContext pc, S3Properties properties,
 				//
 			}
 		}
-		path = prettifyPath(path.substring(atIndex + 1));
-		index = path.indexOf('/');
-		properties.setHost(host);
-		if (index == -1) {
-			if (path.equalsIgnoreCase(S3.DEFAULT_HOST) || path.equalsIgnoreCase(host)) {
-				properties.setHost(path);
-				path = "/";
-			}
-		}
-		else {
-			String _host = path.substring(0, index);
-			if (_host.equalsIgnoreCase(S3.DEFAULT_HOST) || _host.equalsIgnoreCase(host)) {
-				properties.setHost(_host);
-				path = path.substring(index);
+		String srcPath = path;
+		if (atIndex == -1){
+			// no host information, skip parsing, ie. s3://bucketname/ etc
+			path = prettifyPath(path);
+		} else if ( (slashIndex - atIndex) == 1 ){ 
+			// key/secret@/bucket 
+			path = prettifyPath(path.substring(atIndex + 1));
+		} else {
+			int doubleIndex = path.indexOf("://", atIndex); // do we have a http(s)://
+			if (doubleIndex > 0) {
+				// key/secret@http://localhost:9000/bucket 
+				index = path.indexOf('/', doubleIndex+3 );
+				host = path.substring(atIndex+1, index);
+				path = prettifyPath(path.substring(index));
+			} else {
+				path = prettifyPath(path.substring(atIndex + 1));
+				index = path.indexOf('/');
+				if (index == -1) {
+					path = "/";
+				} else if ( (slashIndex - atIndex) != 1 ) { 
+					// key/[email protected]/bucket 
+					host = path.substring(0, index); 
+					path = path.substring(index);
+				}
 			}
 		}
-
+		if (host.endsWith("/")) 
+			host = host.substring(0, host.length() - 1); // strip trailing / after hostname with http://localhost:9000/
+		properties.setHost(host);	
 		// get from system.properties/env.var
 		if (Util.isEmpty(accessKeyId, true)) {
 			if (Util.isEmpty(mappingName)) {
@@ -350,4 +362,4 @@ public Map getArguments() {
 	public char getSeparator() {
 		return '/';
 	}
-}
+}