From 532e77040dac89419a5daf7c7a69fc50e5a85010 Mon Sep 17 00:00:00 2001 From: Harshil Sanjay Jain Date: Wed, 7 Aug 2024 14:33:15 -0400 Subject: [PATCH 01/10] Fix subnet creation --- src/main/java/gyro/azure/network/NetworkResource.java | 7 +++++-- src/main/java/gyro/azure/network/SubnetResource.java | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/gyro/azure/network/NetworkResource.java b/src/main/java/gyro/azure/network/NetworkResource.java index a634f64d..6b51ddc2 100644 --- a/src/main/java/gyro/azure/network/NetworkResource.java +++ b/src/main/java/gyro/azure/network/NetworkResource.java @@ -273,13 +273,16 @@ public void create(GyroUI ui, State state) { withAddressSpace = networkDefWithoutAddress.withAddressSpace(addressSpace); } + withAddressSpace = withAddressSpace.withSubnets(getSubnet().stream() + .collect(Collectors.toMap(SubnetResource::getName, SubnetResource::getAddressPrefix))); + Network network = withAddressSpace .withTags(getTags()) .create(); - network = network.update().withoutSubnet("subnet1").apply(); - copyFrom(network); + + getSubnet().clear(); } @Override diff --git a/src/main/java/gyro/azure/network/SubnetResource.java b/src/main/java/gyro/azure/network/SubnetResource.java index c109df5b..26e0d61e 100644 --- a/src/main/java/gyro/azure/network/SubnetResource.java +++ b/src/main/java/gyro/azure/network/SubnetResource.java @@ -151,6 +151,7 @@ public void copyFrom(Subnet subnet) { RouteTableResource.class, subnet.routeTableId()) : null); setServiceEndpoints(toServiceEndpoints(subnet.servicesWithAccess())); + setId(subnet.innerModel().id()); } @Override From 7347fee20ed30347fabbf824e2a078872b546203 Mon Sep 17 00:00:00 2001 From: Harshil Sanjay Jain Date: Wed, 7 Aug 2024 15:54:14 -0400 Subject: [PATCH 02/10] refresh parent upon subnet creation --- src/main/java/gyro/azure/network/NetworkResource.java | 3 +-- src/main/java/gyro/azure/network/SubnetResource.java | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/gyro/azure/network/NetworkResource.java b/src/main/java/gyro/azure/network/NetworkResource.java index 6b51ddc2..c2b71eab 100644 --- a/src/main/java/gyro/azure/network/NetworkResource.java +++ b/src/main/java/gyro/azure/network/NetworkResource.java @@ -132,6 +132,7 @@ public void setAddressSpaces(Set addressSpaces) { * * @subresource gyro.azure.network.SubnetResource */ + @Required public Set getSubnet() { if (subnet == null) { subnet = new HashSet<>(); @@ -281,8 +282,6 @@ public void create(GyroUI ui, State state) { .create(); copyFrom(network); - - getSubnet().clear(); } @Override diff --git a/src/main/java/gyro/azure/network/SubnetResource.java b/src/main/java/gyro/azure/network/SubnetResource.java index 26e0d61e..2151c060 100644 --- a/src/main/java/gyro/azure/network/SubnetResource.java +++ b/src/main/java/gyro/azure/network/SubnetResource.java @@ -187,6 +187,8 @@ public void create(GyroUI ui, State state) { Network response = updateWithAttach.attach().apply(); setId(response.subnets().get(getName()).id()); + + parent.refresh(); } @Override From 5b9113456d8c9fc68a747e9b7df652e42df13fce Mon Sep 17 00:00:00 2001 From: Harshil Sanjay Jain Date: Thu, 8 Aug 2024 11:51:13 -0400 Subject: [PATCH 03/10] Save state file for containers --- .../gyro/azure/storage/CloudBlobContainerResource.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/gyro/azure/storage/CloudBlobContainerResource.java b/src/main/java/gyro/azure/storage/CloudBlobContainerResource.java index 9b5d098d..7c52a184 100644 --- a/src/main/java/gyro/azure/storage/CloudBlobContainerResource.java +++ b/src/main/java/gyro/azure/storage/CloudBlobContainerResource.java @@ -18,18 +18,18 @@ import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.Set; -import java.util.concurrent.TimeUnit; import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.BlobServiceClient; import com.azure.storage.blob.BlobServiceClientBuilder; +import com.azure.storage.blob.models.BlobContainerAccessPolicies; import com.azure.storage.blob.models.PublicAccessType; import gyro.azure.AzureResource; import gyro.azure.Copyable; import gyro.core.GyroUI; import gyro.core.Type; -import gyro.core.Wait; import gyro.core.resource.Id; import gyro.core.resource.Output; import gyro.core.resource.Resource; @@ -127,7 +127,8 @@ public void setId(String id) { @Override public void copyFrom(BlobContainerClient container) { - setPublicAccess(container.getAccessPolicy().getBlobAccessType().toString()); + setPublicAccess(Optional.ofNullable(container).map(BlobContainerClient::getAccessPolicy).map( + BlobContainerAccessPolicies::getBlobAccessType).map(PublicAccessType::toString).orElse(null)); setName(container.getBlobContainerName()); setMetadata(container.getProperties().getMetadata()); setStorageAccount(findById(StorageAccountResource.class, container.getAccountName())); @@ -154,6 +155,8 @@ public void create(GyroUI ui, State state) { blobContainer = blobContainer(); + state.save(); + blobContainer.setAccessPolicy(PublicAccessType.fromString(getPublicAccess()), null); if (!getMetadata().isEmpty()) { From fa2a33b52013eea85e6d2a0c64f0317317be5394 Mon Sep 17 00:00:00 2001 From: Harshil Sanjay Jain Date: Thu, 8 Aug 2024 13:56:54 -0400 Subject: [PATCH 04/10] Public access shouldn't be required for containers --- .../gyro/azure/storage/CloudBlobContainerResource.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/gyro/azure/storage/CloudBlobContainerResource.java b/src/main/java/gyro/azure/storage/CloudBlobContainerResource.java index 7c52a184..aace10ce 100644 --- a/src/main/java/gyro/azure/storage/CloudBlobContainerResource.java +++ b/src/main/java/gyro/azure/storage/CloudBlobContainerResource.java @@ -157,10 +157,12 @@ public void create(GyroUI ui, State state) { state.save(); - blobContainer.setAccessPolicy(PublicAccessType.fromString(getPublicAccess()), null); + if (getPublicAccess() != null) { + blobContainer.setAccessPolicy(PublicAccessType.fromString(getPublicAccess()), null); - if (!getMetadata().isEmpty()) { - blobContainer.setMetadata(getMetadata()); + if (!getMetadata().isEmpty()) { + blobContainer.setMetadata(getMetadata()); + } } copyFrom(blobContainer); From 6255e9e3e0c3ce08e03e1e222855af2963d10669 Mon Sep 17 00:00:00 2001 From: Harshil Sanjay Jain Date: Thu, 8 Aug 2024 14:10:24 -0400 Subject: [PATCH 05/10] Public access shouldn't be required for containers --- src/main/java/gyro/azure/storage/CloudBlobContainerResource.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/gyro/azure/storage/CloudBlobContainerResource.java b/src/main/java/gyro/azure/storage/CloudBlobContainerResource.java index aace10ce..95e3c3bd 100644 --- a/src/main/java/gyro/azure/storage/CloudBlobContainerResource.java +++ b/src/main/java/gyro/azure/storage/CloudBlobContainerResource.java @@ -77,7 +77,6 @@ public void setName(String name) { /** * The public access of the container. */ - @Required @ValidStrings({ "blob", "container" }) @Updatable public String getPublicAccess() { From 37abdbf48545d1b0b22327266af39e85b35a7683 Mon Sep 17 00:00:00 2001 From: Harshil Sanjay Jain Date: Thu, 8 Aug 2024 17:03:25 -0400 Subject: [PATCH 06/10] enableMultipleStandardLoadBalancers shouldnt have a default value --- .../containerservice/ClusterLoadBalancerProfile.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/main/java/gyro/azure/containerservice/ClusterLoadBalancerProfile.java b/src/main/java/gyro/azure/containerservice/ClusterLoadBalancerProfile.java index f3c5fa8b..431cffbc 100644 --- a/src/main/java/gyro/azure/containerservice/ClusterLoadBalancerProfile.java +++ b/src/main/java/gyro/azure/containerservice/ClusterLoadBalancerProfile.java @@ -66,10 +66,6 @@ public void setEffectiveOutboundIps(List effectiveOutboundIps) { * If set to ``true`` enables multiple standard load balancer. Defaults to ``false``. */ public Boolean getEnableMultipleStandardLoadBalancers() { - if (enableMultipleStandardLoadBalancers == null) { - enableMultipleStandardLoadBalancers = false; - } - return enableMultipleStandardLoadBalancers; } @@ -171,7 +167,9 @@ protected ManagedClusterLoadBalancerProfile toClusterLoadBalancerProfile() { profile.withAllocatedOutboundPorts(getAllocatedOutboundPorts()); } - profile.withEnableMultipleStandardLoadBalancers(getEnableMultipleStandardLoadBalancers()); + if (getEnableMultipleStandardLoadBalancers() != null) { + profile.withEnableMultipleStandardLoadBalancers(getEnableMultipleStandardLoadBalancers()); + } if (getIdleTimeoutInMinutes() != null) { profile.withIdleTimeoutInMinutes(getIdleTimeoutInMinutes()); From 5ff7863592802db59f1a49a2621e8c7b0c45dc58 Mon Sep 17 00:00:00 2001 From: Harshil Sanjay Jain Date: Mon, 12 Aug 2024 11:42:53 -0400 Subject: [PATCH 07/10] Add waits to publuc container creation --- .../storage/CloudBlobContainerResource.java | 36 ++++++++++++++----- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/src/main/java/gyro/azure/storage/CloudBlobContainerResource.java b/src/main/java/gyro/azure/storage/CloudBlobContainerResource.java index 95e3c3bd..f318f36b 100644 --- a/src/main/java/gyro/azure/storage/CloudBlobContainerResource.java +++ b/src/main/java/gyro/azure/storage/CloudBlobContainerResource.java @@ -20,16 +20,20 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.concurrent.TimeUnit; +import com.azure.core.http.rest.Response; import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.BlobServiceClient; import com.azure.storage.blob.BlobServiceClientBuilder; import com.azure.storage.blob.models.BlobContainerAccessPolicies; import com.azure.storage.blob.models.PublicAccessType; +import com.azure.storage.blob.options.BlobContainerCreateOptions; import gyro.azure.AzureResource; import gyro.azure.Copyable; import gyro.core.GyroUI; import gyro.core.Type; +import gyro.core.Wait; import gyro.core.resource.Id; import gyro.core.resource.Output; import gyro.core.resource.Resource; @@ -150,20 +154,34 @@ public boolean refresh() { public void create(GyroUI ui, State state) { BlobContainerClient blobContainer = blobContainer(); - blobContainer.create(); - - blobContainer = blobContainer(); - - state.save(); + BlobContainerCreateOptions blobContainerCreateOptions = new BlobContainerCreateOptions(); if (getPublicAccess() != null) { - blobContainer.setAccessPolicy(PublicAccessType.fromString(getPublicAccess()), null); + blobContainerCreateOptions.setPublicAccessType(PublicAccessType.fromString(getPublicAccess())); + } - if (!getMetadata().isEmpty()) { - blobContainer.setMetadata(getMetadata()); - } + if (!getMetadata().isEmpty()) { + blobContainerCreateOptions.setMetadata(getMetadata()); } + // We add this wait to create as sometimes there is a delay for the StorageAccount to actually become public. + // This results in a 409 error that gets thrown + // Unfortunately th api doesn't really provide a good way to determine if the account is actually public. + Wait.atMost(2, TimeUnit.MINUTES) + .prompt(false) + .checkEvery(10, TimeUnit.SECONDS) + .until(() -> { + Response response = + blobContainer.createIfNotExistsWithResponse(blobContainerCreateOptions, null, null); + + if (response.getStatusCode() == 409) { + return false; + + } else { + return true; + } + }); + copyFrom(blobContainer); } From 4a72a12a853e9b950c9feb4480992bd786f58f25 Mon Sep 17 00:00:00 2001 From: Harshil Sanjay Jain Date: Tue, 13 Aug 2024 15:22:33 -0400 Subject: [PATCH 08/10] container public access should rely on error codes instead of responses --- .../storage/CloudBlobContainerResource.java | 59 +++++++++++-------- 1 file changed, 36 insertions(+), 23 deletions(-) diff --git a/src/main/java/gyro/azure/storage/CloudBlobContainerResource.java b/src/main/java/gyro/azure/storage/CloudBlobContainerResource.java index f318f36b..7e097dae 100644 --- a/src/main/java/gyro/azure/storage/CloudBlobContainerResource.java +++ b/src/main/java/gyro/azure/storage/CloudBlobContainerResource.java @@ -22,15 +22,17 @@ import java.util.Set; import java.util.concurrent.TimeUnit; -import com.azure.core.http.rest.Response; +import com.azure.resourcemanager.storage.models.StorageAccount; import com.azure.storage.blob.BlobContainerClient; import com.azure.storage.blob.BlobServiceClient; import com.azure.storage.blob.BlobServiceClientBuilder; import com.azure.storage.blob.models.BlobContainerAccessPolicies; +import com.azure.storage.blob.models.BlobErrorCode; +import com.azure.storage.blob.models.BlobStorageException; import com.azure.storage.blob.models.PublicAccessType; -import com.azure.storage.blob.options.BlobContainerCreateOptions; import gyro.azure.AzureResource; import gyro.azure.Copyable; +import gyro.core.GyroException; import gyro.core.GyroUI; import gyro.core.Type; import gyro.core.Wait; @@ -154,33 +156,44 @@ public boolean refresh() { public void create(GyroUI ui, State state) { BlobContainerClient blobContainer = blobContainer(); - BlobContainerCreateOptions blobContainerCreateOptions = new BlobContainerCreateOptions(); + try { + blobContainer.create(); - if (getPublicAccess() != null) { - blobContainerCreateOptions.setPublicAccessType(PublicAccessType.fromString(getPublicAccess())); + } catch (BlobStorageException ex) { + System.out.println(ex.getErrorCode()); + System.out.println(ex.getMessage()); + throw new GyroException( + String.format("Could not create container [%s] as it already exists", getName()), ex); } if (!getMetadata().isEmpty()) { - blobContainerCreateOptions.setMetadata(getMetadata()); + blobContainer.setMetadata(getMetadata()); } - // We add this wait to create as sometimes there is a delay for the StorageAccount to actually become public. - // This results in a 409 error that gets thrown - // Unfortunately th api doesn't really provide a good way to determine if the account is actually public. - Wait.atMost(2, TimeUnit.MINUTES) - .prompt(false) - .checkEvery(10, TimeUnit.SECONDS) - .until(() -> { - Response response = - blobContainer.createIfNotExistsWithResponse(blobContainerCreateOptions, null, null); - - if (response.getStatusCode() == 409) { - return false; - - } else { - return true; - } - }); + state.save(); + + if (getPublicAccess() != null) { + StorageAccount refreshedStorageAccount = getStorageAccount().getStorageAccount(); + + if (refreshedStorageAccount != null && refreshedStorageAccount.isBlobPublicAccessAllowed()) { + Wait.atMost(2, TimeUnit.MINUTES) + .prompt(false) + .checkEvery(10, TimeUnit.SECONDS) + .until(() -> { + try { + blobContainer.setAccessPolicy(PublicAccessType.fromString(getPublicAccess()), null); + return true; + + } catch (BlobStorageException ex) { + if (BlobErrorCode.fromString("PublicAccessNotPermitted").equals(ex.getErrorCode())) { + // Retrying as the storage account has not transitioned to public yet + return false; + } + throw ex; + } + }); + } + } copyFrom(blobContainer); } From 863c3a5c2364625f97afb1b4b1a4a3c1432118bc Mon Sep 17 00:00:00 2001 From: Harshil Sanjay Jain Date: Tue, 13 Aug 2024 17:09:21 -0400 Subject: [PATCH 09/10] Remove system.out messages --- .../java/gyro/azure/storage/CloudBlobContainerResource.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/gyro/azure/storage/CloudBlobContainerResource.java b/src/main/java/gyro/azure/storage/CloudBlobContainerResource.java index 7e097dae..1f27835a 100644 --- a/src/main/java/gyro/azure/storage/CloudBlobContainerResource.java +++ b/src/main/java/gyro/azure/storage/CloudBlobContainerResource.java @@ -160,8 +160,6 @@ public void create(GyroUI ui, State state) { blobContainer.create(); } catch (BlobStorageException ex) { - System.out.println(ex.getErrorCode()); - System.out.println(ex.getMessage()); throw new GyroException( String.format("Could not create container [%s] as it already exists", getName()), ex); } From 9f14285f9d8e0586e2be2040a00e93307125ad32 Mon Sep 17 00:00:00 2001 From: Harshil Sanjay Jain Date: Tue, 13 Aug 2024 17:27:51 -0400 Subject: [PATCH 10/10] Save state after creating container --- .../java/gyro/azure/storage/CloudBlobContainerResource.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/gyro/azure/storage/CloudBlobContainerResource.java b/src/main/java/gyro/azure/storage/CloudBlobContainerResource.java index 1f27835a..523a10a0 100644 --- a/src/main/java/gyro/azure/storage/CloudBlobContainerResource.java +++ b/src/main/java/gyro/azure/storage/CloudBlobContainerResource.java @@ -164,6 +164,8 @@ public void create(GyroUI ui, State state) { String.format("Could not create container [%s] as it already exists", getName()), ex); } + state.save(); + if (!getMetadata().isEmpty()) { blobContainer.setMetadata(getMetadata()); }