aws: rds refresh assertion failure

source/extensions/common/aws/BUILD

@@ -122,6 +122,7 @@ envoy_cc_library(
         "//source/common/common:thread_lib",
         "//source/common/config:datasource_lib",
         "//source/common/http:utility_lib",
+        "//source/common/init:manager_lib",
         "//source/common/init:target_lib",
         "//source/common/json:json_loader_lib",
         "//source/common/runtime:runtime_features_lib",

source/extensions/common/aws/credentials_provider.h

@@ -8,6 +8,16 @@
 #include "absl/strings/string_view.h"
 #include "absl/types/optional.h"
 
+namespace Envoy {
+namespace Extensions {
+namespace HttpFilters {
+namespace AwsRequestSigningFilter {
+class FilterConfig;
+}
+} // namespace HttpFilters
+} // namespace Extensions
+} // namespace Envoy
+
 namespace Envoy {
 namespace Extensions {
 namespace Common {
@@ -58,6 +68,8 @@ class Credentials {
  */
 class CredentialsProvider {
 public:
+  using CredentialsPendingCallback = std::function<void(Credentials credentials)>;
+
   virtual ~CredentialsProvider() = default;
 
   /**
@@ -66,6 +78,18 @@ class CredentialsProvider {
    * @return AWS credentials
    */
   virtual Credentials getCredentials() PURE;
+
+  /**
+   * Check if credentials are pending, which supports async credential fetching.
+   *
+   * @return bool true if credentials are pending, false otherwise
+   */
+  virtual bool credentialsPending(
+       ABSL_ATTRIBUTE_UNUSED Envoy::Extensions::HttpFilters::AwsRequestSigningFilter::FilterConfig&
+          config,
+       ABSL_ATTRIBUTE_UNUSED CredentialsPendingCallback&& cb) {
+    return false;
+  }
 };
 
 using CredentialsConstSharedPtr = std::shared_ptr<const Credentials>;

source/extensions/common/aws/credentials_provider_impl.h

@@ -11,6 +11,7 @@
 #include "envoy/event/timer.h"
 #include "envoy/extensions/common/aws/v3/credential_provider.pb.h"
 #include "envoy/http/message.h"
+#include "envoy/init/manager.h"
 #include "envoy/server/factory_context.h"
 
 #include "source/common/common/lock_guard.h"
@@ -143,12 +144,16 @@ class MetadataCredentialsProviderBase : public CachedCredentialsProviderBase {
       std::chrono::seconds initialization_timer);
 
   Credentials getCredentials() override;
-
+  bool credentialsPending(
+      ABSL_ATTRIBUTE_UNUSED Envoy::Extensions::HttpFilters::AwsRequestSigningFilter::FilterConfig&
+          config,
+      ABSL_ATTRIBUTE_UNUSED CredentialsPendingCallback&& cb) override;
   // Get the Metadata credentials cache duration.
   static std::chrono::seconds getCacheDuration();
 
 private:
-  void createCluster(bool new_timer);
+void createCluster(bool new_timer, std::string cluster_name, const envoy::config::cluster::v3::Cluster::DiscoveryType cluster_type, std::string uri );
+  void initializeTlsAndCluster();
 
 protected:
   struct LoadClusterEntryHandleImpl
@@ -244,6 +249,11 @@ class MetadataCredentialsProviderBase : public CachedCredentialsProviderBase {
   std::shared_ptr<MetadataCredentialsProviderStats> stats_;
   // Atomic flag for cluster recreate
   std::atomic<bool> is_creating_ = false;
+  // Are credentials pending?
+  std::atomic<bool> credentials_pending_ = true;
+  // Callbacks list for pending credentials
+  std::vector<CredentialsPendingCallback> credential_pending_callbacks_ = {};
+  Thread::MutexBasicLockable mu_;
 };
 
 /**
@@ -321,22 +331,26 @@ class ContainerCredentialsProvider : public MetadataCredentialsProviderBase,
  * OpenID)
  */
 class WebIdentityCredentialsProvider : public MetadataCredentialsProviderBase,
+                                       public Envoy::Singleton::Instance,
                                        public MetadataFetcher::MetadataReceiver {
 public:
   // token and token_file_path are mutually exclusive. If token is not empty, token_file_path is
   // not used, and vice versa.
   WebIdentityCredentialsProvider(
       Server::Configuration::ServerFactoryContext& context,
-      CreateMetadataFetcherCb create_metadata_fetcher_cb, absl::string_view sts_endpoint,
+      CreateMetadataFetcherCb create_metadata_fetcher_cb, absl::string_view region,
       MetadataFetcher::MetadataReceiver::RefreshState refresh_state,
       std::chrono::seconds initialization_timer,
       const envoy::extensions::common::aws::v3::AssumeRoleWithWebIdentityCredentialProvider&
-          web_identity_config,
-      absl::string_view cluster_name);
+          web_identity_config);
 
   // Following functions are for MetadataFetcher::MetadataReceiver interface
   void onMetadataSuccess(const std::string&& body) override;
   void onMetadataError(Failure reason) override;
+
+  // CredentialsProviderSharedPtr createInstance(const envoy::extensions::common::aws::v3::AssumeRoleWithWebIdentityCredentialProvider&
+  //           web_identity_config,
+  //       absl::string_view sts_endpoint);
 
 private:
   const std::string sts_endpoint_;
@@ -362,6 +376,10 @@ class CredentialsProviderChain : public CredentialsProvider,
   }
 
   Credentials getCredentials() override;
+  bool credentialsPending(
+       Envoy::Extensions::HttpFilters::AwsRequestSigningFilter::FilterConfig&
+          config,
+       CredentialsPendingCallback&& cb) override;
 
 protected:
   std::list<CredentialsProviderSharedPtr> providers_;
@@ -379,13 +397,12 @@ class CredentialsProviderChainFactories {
           credential_file_config = {}) const PURE;
 
   virtual CredentialsProviderSharedPtr createWebIdentityCredentialsProvider(
-      Server::Configuration::ServerFactoryContext& context,
-      CreateMetadataFetcherCb create_metadata_fetcher_cb, absl::string_view sts_endpoint,
+      Server::Configuration::ServerFactoryContext& context, Singleton::Manager& singleton_manager,
+      CreateMetadataFetcherCb create_metadata_fetcher_cb, absl::string_view region,
       MetadataFetcher::MetadataReceiver::RefreshState refresh_state,
       std::chrono::seconds initialization_timer,
       const envoy::extensions::common::aws::v3::AssumeRoleWithWebIdentityCredentialProvider&
-          web_identity_config,
-      absl::string_view cluster_name) const PURE;
+          web_identity_config) const PURE;
 
   virtual CredentialsProviderSharedPtr createContainerCredentialsProvider(
       Api::Api& api, ServerFactoryContextOptRef context, Singleton::Manager& singleton_manager,
@@ -414,13 +431,13 @@ class CustomCredentialsProviderChainFactories {
           credential_file_config = {}) const PURE;
 
   virtual CredentialsProviderSharedPtr createWebIdentityCredentialsProvider(
-      Server::Configuration::ServerFactoryContext& context,
-      CreateMetadataFetcherCb create_metadata_fetcher_cb, absl::string_view sts_endpoint,
+      Server::Configuration::ServerFactoryContext& context, Singleton::Manager& singleton_manager,
+      CreateMetadataFetcherCb create_metadata_fetcher_cb, absl::string_view region,
       MetadataFetcher::MetadataReceiver::RefreshState refresh_state,
       std::chrono::seconds initialization_timer,
       const envoy::extensions::common::aws::v3::AssumeRoleWithWebIdentityCredentialProvider&
-          web_identity_config,
-      absl::string_view cluster_name) const PURE;
+          web_identity_config
+      ) const PURE;
 };
 
 // TODO(nbaws) Add additional providers to the custom chain.
@@ -448,17 +465,13 @@ class CustomCredentialsProviderChain : public CredentialsProviderChain,
   };
 
   CredentialsProviderSharedPtr createWebIdentityCredentialsProvider(
-      Server::Configuration::ServerFactoryContext& context,
-      CreateMetadataFetcherCb create_metadata_fetcher_cb, absl::string_view sts_endpoint,
+      Server::Configuration::ServerFactoryContext& context, Singleton::Manager& singleton_manager,
+      CreateMetadataFetcherCb create_metadata_fetcher_cb, absl::string_view region,
       MetadataFetcher::MetadataReceiver::RefreshState refresh_state,
       std::chrono::seconds initialization_timer,
       const envoy::extensions::common::aws::v3::AssumeRoleWithWebIdentityCredentialProvider&
-          web_identity_config,
-      absl::string_view cluster_name) const override {
-    return std::make_shared<WebIdentityCredentialsProvider>(
-        context, create_metadata_fetcher_cb, sts_endpoint, refresh_state, initialization_timer,
-        web_identity_config, cluster_name);
-  };
+          web_identity_config
+      ) const override;
 };
 
 /**
@@ -534,17 +547,12 @@ class DefaultCredentialsProviderChain : public CredentialsProviderChain,
       std::chrono::seconds initialization_timer, absl::string_view cluster_name) const override;
 
   CredentialsProviderSharedPtr createWebIdentityCredentialsProvider(
-      Server::Configuration::ServerFactoryContext& context,
-      CreateMetadataFetcherCb create_metadata_fetcher_cb, absl::string_view sts_endpoint,
+      Server::Configuration::ServerFactoryContext& context, Singleton::Manager& singleton_manager,
+      CreateMetadataFetcherCb create_metadata_fetcher_cb, absl::string_view region,
       MetadataFetcher::MetadataReceiver::RefreshState refresh_state,
       std::chrono::seconds initialization_timer,
       const envoy::extensions::common::aws::v3::AssumeRoleWithWebIdentityCredentialProvider&
-          web_identity_config,
-      absl::string_view cluster_name) const override {
-    return std::make_shared<WebIdentityCredentialsProvider>(
-        context, create_metadata_fetcher_cb, sts_endpoint, refresh_state, initialization_timer,
-        web_identity_config, cluster_name);
-  }
+          web_identity_config) const override;
 };
 
 using InstanceProfileCredentialsProviderPtr = std::shared_ptr<InstanceProfileCredentialsProvider>;

source/extensions/common/aws/signer.h

@@ -3,6 +3,8 @@
 #include "envoy/common/pure.h"
 #include "envoy/http/message.h"
 
+#include "credentials_provider.h"
+
 namespace Envoy {
 namespace Extensions {
 namespace Common {
@@ -19,8 +21,8 @@ class Signer {
    * @param override_region override the default region that has to be used to sign the request
    * @throws EnvoyException if the request cannot be signed.
    */
-  virtual absl::Status sign(Http::RequestMessage& message, bool sign_body,
-                            const absl::string_view override_region = "") PURE;
+  virtual absl::Status sign(Http::RequestMessage& message, const Credentials credentials,
+                            bool sign_body, const absl::string_view override_region = "") PURE;
 
   /**
    * Sign an AWS request without a payload (empty string used as content hash).
@@ -29,6 +31,7 @@ class Signer {
    * @throws EnvoyException if the request cannot be signed.
    */
   virtual absl::Status signEmptyPayload(Http::RequestHeaderMap& headers,
+                                        const Credentials credentials,
                                         const absl::string_view override_region = "") PURE;
 
   /**
@@ -38,6 +41,7 @@ class Signer {
    * @throws EnvoyException if the request cannot be signed.
    */
   virtual absl::Status signUnsignedPayload(Http::RequestHeaderMap& headers,
+                                           const Credentials credentials,
                                            const absl::string_view override_region = "") PURE;
 
   /**
@@ -47,7 +51,8 @@ class Signer {
    * @param override_region override the default region that has to be used to sign the request
    * @throws EnvoyException if the request cannot be signed.
    */
-  virtual absl::Status sign(Http::RequestHeaderMap& headers, const std::string& content_hash,
+  virtual absl::Status sign(Http::RequestHeaderMap& headers, const Credentials credentials,
+                            const std::string& content_hash,
                             const absl::string_view override_region = "") PURE;
 };
 

source/extensions/common/aws/signer_base_impl.cc

@@ -23,25 +23,29 @@ namespace Extensions {
 namespace Common {
 namespace Aws {
 
-absl::Status SignerBaseImpl::sign(Http::RequestMessage& message, bool sign_body,
-                                  const absl::string_view override_region) {
+absl::Status SignerBaseImpl::sign(Http::RequestMessage& message, const Credentials credentials,
+                                  bool sign_body, const absl::string_view override_region) {
 
   const auto content_hash = createContentHash(message, sign_body);
   auto& headers = message.headers();
-  return sign(headers, content_hash, override_region);
+  return sign(headers, credentials, content_hash, override_region);
 }
 
 absl::Status SignerBaseImpl::signEmptyPayload(Http::RequestHeaderMap& headers,
+                                              const Credentials credentials,
                                               const absl::string_view override_region) {
   headers.setReference(SignatureHeaders::get().ContentSha256,
                        SignatureConstants::HashedEmptyString);
-  return sign(headers, std::string(SignatureConstants::HashedEmptyString), override_region);
+  return sign(headers, credentials, std::string(SignatureConstants::HashedEmptyString),
+              override_region);
 }
 
 absl::Status SignerBaseImpl::signUnsignedPayload(Http::RequestHeaderMap& headers,
+                                                 const Credentials credentials,
                                                  const absl::string_view override_region) {
   headers.setReference(SignatureHeaders::get().ContentSha256, SignatureConstants::UnsignedPayload);
-  return sign(headers, std::string(SignatureConstants::UnsignedPayload), override_region);
+  return sign(headers, credentials, std::string(SignatureConstants::UnsignedPayload),
+              override_region);
 }
 
 // Region support utilities for sigv4a
@@ -54,14 +58,14 @@ void SignerBaseImpl::addRegionQueryParam(
 
 std::string SignerBaseImpl::getRegion() const { return region_; }
 
-absl::Status SignerBaseImpl::sign(Http::RequestHeaderMap& headers, const std::string& content_hash,
+absl::Status SignerBaseImpl::sign(Http::RequestHeaderMap& headers, const Credentials credentials,
+                                  const std::string& content_hash,
                                   const absl::string_view override_region) {
 
   if (!query_string_ && !content_hash.empty()) {
     headers.setReferenceKey(SignatureHeaders::get().ContentSha256, content_hash);
   }
 
-  const auto& credentials = credentials_provider_->getCredentials();
   if (!credentials.accessKeyId() || !credentials.secretAccessKey()) {
     // Empty or "anonymous" credentials are a valid use-case for non-production environments.
     // This behavior matches what the AWS SDK would do.

source/extensions/common/aws/signer_base_impl.h

@@ -63,14 +63,12 @@ using AwsSigningHeaderExclusionVector = std::vector<envoy::type::matcher::v3::St
 class SignerBaseImpl : public Signer, public Logger::Loggable<Logger::Id::aws> {
 public:
   SignerBaseImpl(absl::string_view service_name, absl::string_view region,
-                 const CredentialsProviderSharedPtr& credentials_provider,
                  Server::Configuration::CommonFactoryContext& context,
                  const AwsSigningHeaderExclusionVector& matcher_config,
                  const bool query_string = false,
                  const uint16_t expiration_time = SignatureQueryParameterValues::DefaultExpiration)
       : service_name_(service_name), region_(region),
-        excluded_header_matchers_(defaultMatchers(context)),
-        credentials_provider_(credentials_provider), query_string_(query_string),
+        excluded_header_matchers_(defaultMatchers(context)), query_string_(query_string),
         expiration_time_(expiration_time), time_source_(context.timeSource()),
         long_date_formatter_(std::string(SignatureConstants::LongDateFormat)),
         short_date_formatter_(std::string(SignatureConstants::ShortDateFormat)) {
@@ -81,13 +79,14 @@ class SignerBaseImpl : public Signer, public Logger::Loggable<Logger::Id::aws> {
     }
   }
 
-  absl::Status sign(Http::RequestMessage& message, bool sign_body = false,
+  absl::Status sign(Http::RequestMessage& message, const Credentials credentials,
+                    bool sign_body = false, const absl::string_view override_region = "") override;
+  absl::Status sign(Http::RequestHeaderMap& headers, const Credentials credentials,
+                    const std::string& content_hash,
                     const absl::string_view override_region = "") override;
-  absl::Status sign(Http::RequestHeaderMap& headers, const std::string& content_hash,
-                    const absl::string_view override_region = "") override;
-  absl::Status signEmptyPayload(Http::RequestHeaderMap& headers,
+  absl::Status signEmptyPayload(Http::RequestHeaderMap& headers, const Credentials credentials,
                                 const absl::string_view override_region = "") override;
-  absl::Status signUnsignedPayload(Http::RequestHeaderMap& headers,
+  absl::Status signUnsignedPayload(Http::RequestHeaderMap& headers, const Credentials credentials,
                                    const absl::string_view override_region = "") override;
 
 protected:
@@ -154,7 +153,6 @@ class SignerBaseImpl : public Signer, public Logger::Loggable<Logger::Id::aws> {
       Http::Headers::get().ForwardedFor.get(), Http::Headers::get().ForwardedProto.get(),
       "x-amzn-trace-id"};
   std::vector<Matchers::StringMatcherPtr> excluded_header_matchers_;
-  CredentialsProviderSharedPtr credentials_provider_;
   const bool query_string_;
   const uint16_t expiration_time_;
   TimeSource& time_source_;

source/extensions/common/aws/sigv4_signer_impl.h

@@ -45,13 +45,12 @@ class SigV4SignerImpl : public SignerBaseImpl {
 
 public:
   SigV4SignerImpl(absl::string_view service_name, absl::string_view region,
-                  const CredentialsProviderSharedPtr& credentials_provider,
                   Server::Configuration::CommonFactoryContext& context,
                   const AwsSigningHeaderExclusionVector& matcher_config,
                   const bool query_string = false,
                   const uint16_t expiration_time = SignatureQueryParameterValues::DefaultExpiration)
-      : SignerBaseImpl(service_name, region, credentials_provider, context, matcher_config,
-                       query_string, expiration_time) {}
+      : SignerBaseImpl(service_name, region, context, matcher_config, query_string,
+                       expiration_time) {}
 
 private:
   std::string createCredentialScope(const absl::string_view short_date,

source/extensions/common/aws/sigv4a_signer_impl.h

@@ -57,12 +57,11 @@ class SigV4ASignerImpl : public SignerBaseImpl {
 public:
   SigV4ASignerImpl(
       absl::string_view service_name, absl::string_view region,
-      const CredentialsProviderSharedPtr& credentials_provider,
       Server::Configuration::CommonFactoryContext& context,
       const AwsSigningHeaderExclusionVector& matcher_config, const bool query_string = false,
       const uint16_t expiration_time = SignatureQueryParameterValues::DefaultExpiration)
-      : SignerBaseImpl(service_name, region, credentials_provider, context, matcher_config,
-                       query_string, expiration_time) {}
+      : SignerBaseImpl(service_name, region, context, matcher_config, query_string,
+                       expiration_time) {}
 
 private:
   void addRegionHeader(Http::RequestHeaderMap& headers,