From 5859ef52685de2cfa5130cc9b6ab81b5ba007995 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Sun, 3 Dec 2023 20:31:58 +0800 Subject: [PATCH] out_azure_blob: add support for Azure Blob SAS authentication Sometimes user cannot use shared key authentication. So, this patch support authenticate Azure Blob Storage with shared access signatures. Here is the document about shared access signatures: https://learn.microsoft.com/en-us/azure/storage/common/storage-sas-overview This patch add two configuration parameters for out_azure_blob plugin: auth_type and sas_token. - auth_type, default value is "key", it can be "key" or "sas". - sas_token, default value is NULL, it is the SAS token, it is required when auth_type is "sas". Signed-off-by: Yi Lin --- plugins/out_azure_blob/azure_blob.c | 23 ++++++++ plugins/out_azure_blob/azure_blob.h | 6 +++ .../out_azure_blob/azure_blob_appendblob.c | 4 ++ plugins/out_azure_blob/azure_blob_blockblob.c | 14 +++++ plugins/out_azure_blob/azure_blob_conf.c | 54 +++++++++++++++---- plugins/out_azure_blob/azure_blob_http.c | 22 ++++---- plugins/out_azure_blob/azure_blob_uri.c | 8 +++ 7 files changed, 111 insertions(+), 20 deletions(-) diff --git a/plugins/out_azure_blob/azure_blob.c b/plugins/out_azure_blob/azure_blob.c index 3f539826db6..b5be3dd3495 100644 --- a/plugins/out_azure_blob/azure_blob.c +++ b/plugins/out_azure_blob/azure_blob.c @@ -200,6 +200,12 @@ static int send_blob(struct flb_config *config, return FLB_OK; } else if (c->resp.status == 404) { + /* delete "&sig=..." in the c->uri for security */ + char *p = strstr(c->uri, "&sig="); + if (p) { + *p = '\0'; + } + flb_plg_info(ctx->ins, "blob not found: %s", c->uri); flb_http_client_destroy(c); return CREATE_BLOB; @@ -269,6 +275,11 @@ static int create_blob(struct flb_azure_blob *ctx, char *name) } if (c->resp.status == 201) { + /* delete "&sig=..." in the c->uri for security */ + char *p = strstr(c->uri, "&sig="); + if (p) { + *p = '\0'; + } flb_plg_info(ctx->ins, "blob created successfully: %s", c->uri); } else { @@ -572,6 +583,18 @@ static struct flb_config_map config_map[] = { "Name of the key that will have the record timestamp" }, + { + FLB_CONFIG_MAP_STR, "auth_type", "key", + 0, FLB_TRUE, offsetof(struct flb_azure_blob, auth_type), + "Set the auth type: key or sas" + }, + + { + FLB_CONFIG_MAP_STR, "sas_token", NULL, + 0, FLB_TRUE, offsetof(struct flb_azure_blob, sas_token), + "Azure Blob SAS token" + }, + /* EOF */ {0} }; diff --git a/plugins/out_azure_blob/azure_blob.h b/plugins/out_azure_blob/azure_blob.h index 5cf8a29279a..afa3a67132d 100644 --- a/plugins/out_azure_blob/azure_blob.h +++ b/plugins/out_azure_blob/azure_blob.h @@ -41,6 +41,9 @@ #define AZURE_BLOB_APPENDBLOB 0 #define AZURE_BLOB_BLOCKBLOB 1 +#define AZURE_BLOB_AUTH_KEY 0 +#define AZURE_BLOB_AUTH_SAS 1 + struct flb_azure_blob { int auto_create_container; int emulator_mode; @@ -53,11 +56,14 @@ struct flb_azure_blob { flb_sds_t endpoint; flb_sds_t path; flb_sds_t date_key; + flb_sds_t auth_type; + flb_sds_t sas_token; /* * Internal use */ int btype; /* blob type */ + int atype; /* auth type */ flb_sds_t real_endpoint; flb_sds_t base_uri; flb_sds_t shared_key_prefix; diff --git a/plugins/out_azure_blob/azure_blob_appendblob.c b/plugins/out_azure_blob/azure_blob_appendblob.c index 2d9a8217103..e63ba2a4f90 100644 --- a/plugins/out_azure_blob/azure_blob_appendblob.c +++ b/plugins/out_azure_blob/azure_blob_appendblob.c @@ -40,5 +40,9 @@ flb_sds_t azb_append_blob_uri(struct flb_azure_blob *ctx, char *tag) flb_sds_printf(&uri, "/%s?comp=appendblock", tag); } + if (ctx->atype == AZURE_BLOB_AUTH_SAS && ctx->sas_token) { + flb_sds_printf(&uri, "&%s", ctx->sas_token); + } + return uri; } diff --git a/plugins/out_azure_blob/azure_blob_blockblob.c b/plugins/out_azure_blob/azure_blob_blockblob.c index a9b0e4a286e..499678ad3ee 100644 --- a/plugins/out_azure_blob/azure_blob_blockblob.c +++ b/plugins/out_azure_blob/azure_blob_blockblob.c @@ -65,6 +65,10 @@ flb_sds_t azb_block_blob_uri(struct flb_azure_blob *ctx, char *tag, tag, ms, ext, encoded_blockid); } + if (ctx->atype == AZURE_BLOB_AUTH_SAS && ctx->sas_token) { + flb_sds_printf(&uri, "&%s", ctx->sas_token); + } + flb_sds_destroy(encoded_blockid); return uri; } @@ -95,6 +99,10 @@ flb_sds_t azb_block_blob_uri_commit(struct flb_azure_blob *ctx, flb_sds_printf(&uri, "/%s.%" PRIu64 "%s?comp=blocklist", tag, ms, ext); } + if (ctx->atype == AZURE_BLOB_AUTH_SAS && ctx->sas_token) { + flb_sds_printf(&uri, "&%s", ctx->sas_token); + } + return uri; } @@ -214,6 +222,12 @@ int azb_block_blob_commit(struct flb_azure_blob *ctx, char *blockid, char *tag, return FLB_OK; } else if (c->resp.status == 404) { + /* delete "&sig=..." in the c->uri for security */ + char *p = strstr(c->uri, "&sig="); + if (p) { + *p = '\0'; + } + flb_plg_info(ctx->ins, "blob not found: %s", c->uri); flb_http_client_destroy(c); flb_upstream_conn_release(u_conn); diff --git a/plugins/out_azure_blob/azure_blob_conf.c b/plugins/out_azure_blob/azure_blob_conf.c index 4437a6d2d5a..ddf4ed8605a 100644 --- a/plugins/out_azure_blob/azure_blob_conf.c +++ b/plugins/out_azure_blob/azure_blob_conf.c @@ -85,8 +85,39 @@ struct flb_azure_blob *flb_azure_blob_conf_create(struct flb_output_instance *in return NULL; } + /* Set Auth type */ + tmp = (char *) flb_output_get_property("auth_type", ins); + if (!tmp) { + ctx->atype = AZURE_BLOB_AUTH_KEY; + } + else { + if (strcasecmp(tmp, "key") == 0) { + ctx->atype = AZURE_BLOB_AUTH_KEY; + } + else if (strcasecmp(tmp, "sas") == 0) { + ctx->atype = AZURE_BLOB_AUTH_SAS; + } + else { + flb_plg_error(ctx->ins, "invalid auth_type value '%s'", tmp); + return NULL; + } + } + if (ctx->atype == AZURE_BLOB_AUTH_KEY && !ctx->shared_key) { + flb_plg_error(ctx->ins, "'shared_key' has not been set"); + return NULL; + } + if (ctx->atype == AZURE_BLOB_AUTH_SAS) { + if (!ctx->sas_token) { + flb_plg_error(ctx->ins, "'sas_token' has not been set"); + return NULL; + } + if (ctx->sas_token[0] == '?') { + ctx->sas_token++; + } + } + /* If the shared key is set decode it */ - if (ctx->shared_key) { + if (ctx->atype == AZURE_BLOB_AUTH_KEY && ctx->shared_key) { ret = set_shared_key(ctx); if (ret == -1) { return NULL; @@ -196,12 +227,14 @@ struct flb_azure_blob *flb_azure_blob_conf_create(struct flb_output_instance *in } /* Prepare shared key buffer */ - ctx->shared_key_prefix = flb_sds_create_size(256); - if (!ctx->shared_key_prefix) { - flb_plg_error(ctx->ins, "cannot create shared key prefix"); - return NULL; + if (ctx->atype == AZURE_BLOB_AUTH_KEY) { + ctx->shared_key_prefix = flb_sds_create_size(256); + if (!ctx->shared_key_prefix) { + flb_plg_error(ctx->ins, "cannot create shared key prefix"); + return NULL; + } + flb_sds_printf(&ctx->shared_key_prefix, "SharedKey %s:", ctx->account_name); } - flb_sds_printf(&ctx->shared_key_prefix, "SharedKey %s:", ctx->account_name); /* Sanitize path: remove any ending slash */ if (ctx->path) { @@ -211,11 +244,12 @@ struct flb_azure_blob *flb_azure_blob_conf_create(struct flb_output_instance *in } flb_plg_info(ctx->ins, - "account_name=%s, container_name=%s, blob_type=%s, emulator_mode=%s, endpoint=%s", + "account_name=%s, container_name=%s, blob_type=%s, emulator_mode=%s, endpoint=%s, auth_type=%s", ctx->account_name, ctx->container_name, - ctx->btype == AZURE_BLOB_APPENDBLOB ? "appendblob": "blockblob", - ctx->emulator_mode ? "yes": "no", - ctx->real_endpoint ? ctx->real_endpoint: "no"); + ctx->btype == AZURE_BLOB_APPENDBLOB ? "appendblob" : "blockblob", + ctx->emulator_mode ? "yes" : "no", + ctx->real_endpoint ? ctx->real_endpoint : "no", + ctx->atype == AZURE_BLOB_AUTH_KEY ? "key" : "sas"); return ctx; } diff --git a/plugins/out_azure_blob/azure_blob_http.c b/plugins/out_azure_blob/azure_blob_http.c index 5ac81a9a126..9ddced427af 100644 --- a/plugins/out_azure_blob/azure_blob_http.c +++ b/plugins/out_azure_blob/azure_blob_http.c @@ -339,20 +339,22 @@ int azb_http_client_setup(struct flb_azure_blob *ctx, struct flb_http_client *c, /* Azure header: x-ms-version */ flb_http_add_header(c, "x-ms-version", 12, "2019-12-12", 10); - can_req = azb_http_canonical_request(ctx, c, content_length, content_type, - content_encoding); + if (ctx->atype == AZURE_BLOB_AUTH_KEY) { + can_req = azb_http_canonical_request(ctx, c, content_length, content_type, + content_encoding); - auth = flb_sds_create_size(64 + flb_sds_len(can_req)); + auth = flb_sds_create_size(64 + flb_sds_len(can_req)); - flb_sds_cat(auth, ctx->shared_key_prefix, flb_sds_len(ctx->shared_key_prefix)); - flb_sds_cat(auth, can_req, flb_sds_len(can_req)); + flb_sds_cat(auth, ctx->shared_key_prefix, flb_sds_len(ctx->shared_key_prefix)); + flb_sds_cat(auth, can_req, flb_sds_len(can_req)); - /* Azure header: authorization */ - flb_http_add_header(c, "Authorization", 13, auth, flb_sds_len(auth)); + /* Azure header: authorization */ + flb_http_add_header(c, "Authorization", 13, auth, flb_sds_len(auth)); - /* Release buffers */ - flb_sds_destroy(can_req); - flb_sds_destroy(auth); + /* Release buffers */ + flb_sds_destroy(can_req); + flb_sds_destroy(auth); + } /* Set callback context to the HTTP client context */ flb_http_set_callback_context(c, ctx->ins->callback); diff --git a/plugins/out_azure_blob/azure_blob_uri.c b/plugins/out_azure_blob/azure_blob_uri.c index c7a05e286aa..a63fc156980 100644 --- a/plugins/out_azure_blob/azure_blob_uri.c +++ b/plugins/out_azure_blob/azure_blob_uri.c @@ -127,6 +127,10 @@ flb_sds_t azb_uri_ensure_or_create_container(struct flb_azure_blob *ctx) } flb_sds_printf(&uri, "?restype=container"); + if (ctx->atype == AZURE_BLOB_AUTH_SAS && ctx->sas_token) { + flb_sds_printf(&uri, "&%s", ctx->sas_token); + } + return uri; } @@ -146,5 +150,9 @@ flb_sds_t azb_uri_create_blob(struct flb_azure_blob *ctx, char *tag) flb_sds_printf(&uri, "/%s", tag); } + if (ctx->atype == AZURE_BLOB_AUTH_SAS && ctx->sas_token) { + flb_sds_printf(&uri, "?%s", ctx->sas_token); + } + return uri; }