Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Recompute TTL values on each get #63

Merged
merged 1 commit into from
Oct 27, 2023
Merged

Recompute TTL values on each get #63

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 3 additions & 8 deletions secret-provider/src/main/scala/io/lenses/connect/secrets/package.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,13 @@
package io.lenses.connect.secrets

import com.typesafe.scalalogging.StrictLogging
import org.apache.kafka.common.config.ConfigData
import org.apache.kafka.connect.errors.ConnectException

import java.io.File
import java.io.FileOutputStream
import java.time.OffsetDateTime
import java.util.Base64
import scala.collection.mutable
import scala.jdk.CollectionConverters._
import scala.util.Failure
import scala.util.Success
import scala.util.Try
Expand Down Expand Up @@ -150,7 +148,7 @@ package object connect extends StrictLogging {
//calculate the min expiry for secrets and return the configData and expiry
def getSecretsAndExpiry(
secrets: Map[String, (String, Option[OffsetDateTime])],
): (Option[OffsetDateTime], ConfigData) = {
): (Option[OffsetDateTime], Map[String, String]) = {
val expiryList = mutable.ListBuffer.empty[OffsetDateTime]

val data = secrets
Expand All @@ -159,15 +157,12 @@ package object connect extends StrictLogging {
expiry.foreach(e => expiryList.append(e))
(key, value)
})
.asJava

if (expiryList.isEmpty) {
(None, new ConfigData(data))
(None, data)
} else {
val minExpiry = expiryList.min
val ttl = minExpiry.toInstant.toEpochMilli - OffsetDateTime.now.toInstant
.toEpochMilli
(Some(minExpiry), new ConfigData(data, ttl))
(Some(minExpiry), data)
}
}

Expand Down
37 changes: 15 additions & 22 deletions secret-provider/src/main/scala/io/lenses/connect/secrets/providers/AzureSecretProvider.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@

package io.lenses.connect.secrets.providers

import java.time.Duration

import com.azure.core.credential.TokenCredential
import com.azure.security.keyvault.secrets.SecretClient
import com.azure.security.keyvault.secrets.SecretClientBuilder
Expand All @@ -25,7 +27,7 @@ class AzureSecretProvider() extends ConfigProvider with AzureHelper {
private var rootDir: String = _
private var credentials: Option[TokenCredential] = None
val clientMap: mutable.Map[String, SecretClient] = mutable.Map.empty
val cache = mutable.Map.empty[String, (Option[OffsetDateTime], ConfigData)]
val cache = mutable.Map.empty[String, (Option[OffsetDateTime], Map[String, String])]

// configure the vault client
override def configure(configs: util.Map[String, _]): Unit = {
Expand Down Expand Up @@ -62,32 +64,19 @@ class AzureSecretProvider() extends ConfigProvider with AzureHelper {

clientMap += (keyVaultUrl -> client)

val now = OffsetDateTime.now()

val (expiry, data) = cache.get(keyVaultUrl) match {
case Some((expiresAt, data)) =>
// we have all the keys and are before the expiry
val now = OffsetDateTime.now()

if (
keys.asScala.subsetOf(data.data().asScala.keySet) && (expiresAt
keys.asScala.subsetOf(data.keySet) && expiresAt
.getOrElse(now.plusSeconds(1))
.isAfter(now))
.isAfter(now)
) {
logger.info("Fetching secrets from cache")
(
expiresAt,
new ConfigData(
data
.data()
.asScala
.view
.filter {
case (k, _) => keys.contains(k)
}
.toMap
.asJava,
data.ttl(),
),
)
(expiresAt, data.view.filterKeys(k => keys.contains(k)).toMap)
} else {
// missing some or expired so reload
getSecretsAndExpiry(getSecrets(client, keys.asScala.toSet))
Expand All @@ -97,9 +86,13 @@ class AzureSecretProvider() extends ConfigProvider with AzureHelper {
getSecretsAndExpiry(getSecrets(client, keys.asScala.toSet))
}

expiry.foreach(exp => logger.info(s"Min expiry for TTL set to [${exp.toString}]"))
cache += (keyVaultUrl -> (expiry, data))
data
var ttl = 0L
expiry.foreach { exp =>
ttl = Duration.between(now, exp).toMillis
logger.info(s"Min expiry for TTL set to [${exp.toString}]")
}
cache.put(keyVaultUrl, (expiry, data))
new ConfigData(data.asJava, ttl)
}

override def close(): Unit = {}
Expand Down
7 changes: 3 additions & 4 deletions ...provider/src/test/scala/io/lenses/connect/secrets/providers/AzureSecretProviderTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ import io.lenses.connect.secrets.config.AzureProviderSettings
import io.lenses.connect.secrets.connect
import io.lenses.connect.secrets.connect.AuthMode
import org.apache.kafka.common.config.provider.ConfigProvider
import org.apache.kafka.common.config.ConfigData
import org.apache.kafka.common.config.ConfigTransformer
import org.apache.kafka.connect.errors.ConnectException
import org.mockito.Mockito.when
Expand Down Expand Up @@ -236,7 +235,7 @@ class AzureSecretProviderTest extends AnyWordSpec with Matchers with MockitoSuga
// poke in the mocked client
provider.clientMap += (s"https://$secretPath" -> client)
val now = OffsetDateTime.now().plusMinutes(10)
val cachedData = new ConfigData(Map(secretKey -> secretPath).asJava)
val cachedData = Map(secretKey -> secretPath)
val cached = (Some(now), cachedData)

// add to cache
Expand Down Expand Up @@ -283,7 +282,7 @@ class AzureSecretProviderTest extends AnyWordSpec with Matchers with MockitoSuga
provider.clientMap += (vaultUrl -> client)
//put expiry of cache 1 second behind
val now = OffsetDateTime.now().minusSeconds(1)
val cachedData = new ConfigData(Map(secretKey -> secretPath).asJava)
val cachedData = Map(secretKey -> secretPath)
val cached = (Some(now), cachedData)

// add to cache
Expand Down Expand Up @@ -332,7 +331,7 @@ class AzureSecretProviderTest extends AnyWordSpec with Matchers with MockitoSuga
provider.clientMap += (vaultUrl -> client)
//put expiry of cache 1 second behind
val now = OffsetDateTime.now()
val cachedData = new ConfigData(Map("old-key" -> secretPath).asJava)
val cachedData = Map("old-key" -> secretPath)
val cached = (Some(now), cachedData)

// add to cache
Expand Down
Loading