From ec8e564abaf33f8554e24b310d62e85c368351cc Mon Sep 17 00:00:00 2001 From: Shashi Santosh Date: Thu, 12 Apr 2018 17:06:44 +0530 Subject: [PATCH 01/15] ZCS-4021:Implementation to extract jwt from Auth header bearer --- .../modules/ngx_http_upstream_zmauth_module.c | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/thirdparty/nginx/nginx-1.7.1-zimbra/src/http/modules/ngx_http_upstream_zmauth_module.c b/thirdparty/nginx/nginx-1.7.1-zimbra/src/http/modules/ngx_http_upstream_zmauth_module.c index 76ef04a7e..1f090fb6a 100644 --- a/thirdparty/nginx/nginx-1.7.1-zimbra/src/http/modules/ngx_http_upstream_zmauth_module.c +++ b/thirdparty/nginx/nginx-1.7.1-zimbra/src/http/modules/ngx_http_upstream_zmauth_module.c @@ -1043,6 +1043,44 @@ ngx_field_from_zmauthtoken(ngx_log_t *log, ngx_pool_t *pool, return f; } +/* extract the jwt token from Header*/ +static ngx_flag_t +ngx_get_authheader_bearer(ngx_log_t *log, ngx_pool_t *pool, + ngx_http_headers_in_t *headers_in, ngx_str_t *value) +{ + ngx_log_debug0 (NGX_LOG_DEBUG_HTTP, log, 0, + "zmauth: ngx_get_authheader_bearer entered"); + + ngx_str_t authValue; + ngx_str_t token; + + authValue = headers_in->authorization->value; + + ngx_log_debug1 (NGX_LOG_DEBUG_HTTP, log, 0, "zmauth: ngx_get_authheader_bearer token:%V",&authValue); + + if (authValue.len >= sizeof("Bearer ") - 1 + && ngx_memcmp(authValue.data,"Bearer ",sizeof("Bearer ")-1) + == 0) + { + ngx_log_debug0 (NGX_LOG_DEBUG_HTTP, log, 0, "zmauth: ngx_get_authheader_bearer Bearer token found"); + + token = authValue; + token.data += (sizeof("Bearer ") - 1); + token.len -= (sizeof("Bearer ") - 1); + + ngx_log_debug1 (NGX_LOG_DEBUG_HTTP, log, 0, "zmauth: ngx_get_authheader_bearer Bearer token found:%V",&token); + *value = token; + + return 1; + } + + ngx_log_debug0 (NGX_LOG_DEBUG_HTTP, log, 0, + "zmauth: ngx_get_authheader_bearer exit"); + + return 0; +} + + static void * ngx_http_upstream_zmauth_create_srv_conf(ngx_conf_t *cf) { ngx_http_upstream_zmauth_srv_conf_t *zscf; From e4be3536becd95ce47556e592bcad778ce566a52 Mon Sep 17 00:00:00 2001 From: Shashi Santosh Date: Thu, 12 Apr 2018 17:33:54 +0530 Subject: [PATCH 02/15] ZCS-4021:Implementation to extract sub data field from payload --- .../modules/ngx_http_upstream_zmauth_module.c | 66 ++++++++++++++++++- 1 file changed, 64 insertions(+), 2 deletions(-) diff --git a/thirdparty/nginx/nginx-1.7.1-zimbra/src/http/modules/ngx_http_upstream_zmauth_module.c b/thirdparty/nginx/nginx-1.7.1-zimbra/src/http/modules/ngx_http_upstream_zmauth_module.c index 1f090fb6a..fb46f2bda 100644 --- a/thirdparty/nginx/nginx-1.7.1-zimbra/src/http/modules/ngx_http_upstream_zmauth_module.c +++ b/thirdparty/nginx/nginx-1.7.1-zimbra/src/http/modules/ngx_http_upstream_zmauth_module.c @@ -1071,7 +1071,7 @@ ngx_get_authheader_bearer(ngx_log_t *log, ngx_pool_t *pool, ngx_log_debug1 (NGX_LOG_DEBUG_HTTP, log, 0, "zmauth: ngx_get_authheader_bearer Bearer token found:%V",&token); *value = token; - return 1; + return 1; } ngx_log_debug0 (NGX_LOG_DEBUG_HTTP, log, 0, @@ -1081,6 +1081,68 @@ ngx_get_authheader_bearer(ngx_log_t *log, ngx_pool_t *pool, } +/* extract sub data field from payload */ +static ngx_flag_t +ngx_claimdata_from_payload(ngx_log_t *log, ngx_pool_t *pool, + ngx_str_t *payload, ngx_str_t *value) { + + ngx_flag_t f; + ngx_log_debug0 (NGX_LOG_DEBUG_HTTP, log, 0, + "zmauth: ngx_claimdata_from_payload entered"); + + ngx_str_t pattern = ngx_string("\"sub\":[\s]*\"(.*)\"[\s]*"); + ngx_regex_compile_t rgc; + u_char errstr[NGX_MAX_CONF_ERRSTR]; + ngx_memzero(&rgc, sizeof(ngx_regex_compile_t)); + + rgc.pattern = pattern; + rgc.pool = pool; + rgc.err.len = NGX_MAX_CONF_ERRSTR; + rgc.err.data = errstr; + + + if (ngx_regex_compile(&rgc) != NGX_OK) + { + ngx_log_error(NGX_LOG_INFO, log, 0,"zmauth: ngx_claimdata_from_payload regex compilation failed.Error=====%s",errstr); + return 0; + } + + int captures[30]; + ngx_int_t n; + + n = ngx_regex_exec(rgc.regex, payload, captures, 30); + if (n >= 0) + { + /* string matches expression */ + ngx_log_debug0(NGX_LOG_DEBUG_ZIMBRA, log, 0, "zmauth: ngx_claimdata_from_payload match found"); + ngx_str_t subData; + subData.data = payload->data + captures[2]; + subData.len = captures[3] - captures[2]; + ngx_log_debug1(NGX_LOG_DEBUG_ZIMBRA, log, 0, "zmauth: ngx_claimdata_from_payload subData is %V",&subData); + *value = subData; + return 1; + } + else if (n == NGX_REGEX_NO_MATCHED) + { + /* no match was found */ + ngx_log_debug0(NGX_LOG_DEBUG_ZIMBRA, log, 0, "zmauth: ngx_claimdata_from_payload no match found"); + return 0; + } + else + { + /* some error */ + ngx_log_debug0(NGX_LOG_DEBUG_ZIMBRA, log, 0, "zmauth: ngx_claimdata_from_payload no match found due to some error."); + return 0; + } + + ngx_log_debug0 (NGX_LOG_DEBUG_HTTP, log, 0, + "zmauth: ngx_claimdata_from_payload exit"); + + return 0; +} + + + static void * ngx_http_upstream_zmauth_create_srv_conf(ngx_conf_t *cf) { ngx_http_upstream_zmauth_srv_conf_t *zscf; @@ -1459,6 +1521,7 @@ zmauth_check_caldav(ngx_http_request_t *r, void **extra) { return f; } + /* examine request cookies for ZM_AUTH_TOKEN and extract route if so */ static ngx_flag_t zmauth_check_authtoken(ngx_http_request_t *r, ngx_str_t* field, @@ -1523,7 +1586,6 @@ zmauth_check_authtoken(ngx_http_request_t *r, ngx_str_t* field, return f; } - /* examine request cookies for ZM_ADMIN_AUTH_TOKEN and extract route if so */ static ngx_flag_t zmauth_check_admin_authtoken(ngx_http_request_t *r, ngx_str_t* field, From 9f8100a2d3eae803716fedde5d09639a9246c275 Mon Sep 17 00:00:00 2001 From: Shashi Santosh Date: Thu, 12 Apr 2018 17:42:12 +0530 Subject: [PATCH 03/15] ZCS-4021:Implementation to extract claim data from jwt token --- .../modules/ngx_http_upstream_zmauth_module.c | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/thirdparty/nginx/nginx-1.7.1-zimbra/src/http/modules/ngx_http_upstream_zmauth_module.c b/thirdparty/nginx/nginx-1.7.1-zimbra/src/http/modules/ngx_http_upstream_zmauth_module.c index fb46f2bda..9943f93cb 100644 --- a/thirdparty/nginx/nginx-1.7.1-zimbra/src/http/modules/ngx_http_upstream_zmauth_module.c +++ b/thirdparty/nginx/nginx-1.7.1-zimbra/src/http/modules/ngx_http_upstream_zmauth_module.c @@ -1080,6 +1080,75 @@ ngx_get_authheader_bearer(ngx_log_t *log, ngx_pool_t *pool, return 0; } +/* extract claim data from jwt token */ +static ngx_flag_t +ngx_claimdata_from_jwttoken(ngx_log_t *log, ngx_pool_t *pool, + ngx_str_t *authtoken, ngx_str_t *field, ngx_str_t *value) { + + ngx_flag_t f; + ngx_log_debug0 (NGX_LOG_DEBUG_HTTP, log, 0, + "zmauth: ngx_claimdata_from_jwttoken entered"); + + ngx_str_t pattern = ngx_string("(?<=\\.)(.*?)(?=\\.)"); + ngx_regex_compile_t rgc; + u_char errstr[NGX_MAX_CONF_ERRSTR]; + ngx_memzero(&rgc, sizeof(ngx_regex_compile_t)); + + rgc.pattern = pattern; + rgc.pool = pool; + rgc.err.len = NGX_MAX_CONF_ERRSTR; + rgc.err.data = errstr; + + + if (ngx_regex_compile(&rgc) != NGX_OK) + { + ngx_log_error(NGX_LOG_INFO, log, 0,"zmauth: ngx_claimdata_from_jwttoken regex compilation failed.Error=====%s",errstr); + return 0; + } + + int captures[30]; + ngx_int_t n; + + n = ngx_regex_exec(rgc.regex, authtoken, captures, 30); + if (n >= 0) + { + /* string matches expression */ + ngx_log_debug0(NGX_LOG_DEBUG_ZIMBRA, log, 0, "zmauth: ngx_claimdata_from_jwttoken match found"); + ngx_str_t encodedPayload; + encodedPayload.data = authtoken->data + captures[0]; + encodedPayload.len = captures[1] - captures[0]; + ngx_log_debug1(NGX_LOG_DEBUG_ZIMBRA, log, 0, "zmauth: ngx_claimdata_from_jwttoken payload is %V",&encodedPayload); + + ngx_str_t urlDecodedPayload; + if (ngx_decode_base64url(&urlDecodedPayload, &encodedPayload) != NGX_OK) + { + ngx_log_error(NGX_LOG_INFO, log, 0,"zmauth: ngx_claimdata_from_jwttoken base64url decoding failed"); + return 0; + } + ngx_log_debug1(NGX_LOG_DEBUG_ZIMBRA, log, 0, "zmauth: ngx_claimdata_from_jwttoken base64url decoded payload is %V",&urlDecodedPayload); + + int ret = ngx_claimdata_from_payload(log,pool,&urlDecodedPayload,value); + return ret; + } + else if (n == NGX_REGEX_NO_MATCHED) + { + /* no match was found */ + ngx_log_debug0(NGX_LOG_DEBUG_ZIMBRA, log, 0, "zmauth: ngx_claimdata_from_jwttoken no match found"); + return 0; + } + else + { + /* some error */ + ngx_log_debug0(NGX_LOG_DEBUG_ZIMBRA, log, 0, "zmauth: ngx_claimdata_from_jwttoken no match found due to some error."); + return 0; + } + + ngx_log_debug0 (NGX_LOG_DEBUG_HTTP, log, 0, + "zmauth: ngx_claimdata_from_jwttoken exit"); + + return 0; +} + /* extract sub data field from payload */ static ngx_flag_t From 685b45be7db0d97a25c67026243140382ec9a81f Mon Sep 17 00:00:00 2001 From: Shashi Santosh Date: Thu, 12 Apr 2018 17:45:43 +0530 Subject: [PATCH 04/15] ZCS-4021:Implementation to examine request header and query parameter for jwt token and extract route if so --- .../modules/ngx_http_upstream_zmauth_module.c | 119 +++++++++++++++--- .../modules/ngx_http_upstream_zmauth_module.h | 3 +- 2 files changed, 105 insertions(+), 17 deletions(-) diff --git a/thirdparty/nginx/nginx-1.7.1-zimbra/src/http/modules/ngx_http_upstream_zmauth_module.c b/thirdparty/nginx/nginx-1.7.1-zimbra/src/http/modules/ngx_http_upstream_zmauth_module.c index 9943f93cb..8addfb73f 100644 --- a/thirdparty/nginx/nginx-1.7.1-zimbra/src/http/modules/ngx_http_upstream_zmauth_module.c +++ b/thirdparty/nginx/nginx-1.7.1-zimbra/src/http/modules/ngx_http_upstream_zmauth_module.c @@ -60,6 +60,7 @@ static ngx_flag_t zmauth_check_activesync(ngx_http_request_t *r, void **extra); static ngx_flag_t zmauth_check_ews(ngx_http_request_t *r, void **extra); static ngx_flag_t zmauth_check_caldav(ngx_http_request_t *r, void **extra); static ngx_flag_t zmauth_check_authtoken(ngx_http_request_t *r, ngx_str_t *field, void **extra); +static ngx_flag_t zmauth_check_jwttoken(ngx_http_request_t *r, ngx_str_t *field, void **extra); static ngx_flag_t zmauth_check_admin_authtoken(ngx_http_request_t *r, ngx_str_t *field, void **extra); static zmroutetype_t zmauth_check_uri(ngx_http_request_t *r, void **extra); @@ -76,12 +77,19 @@ static void zmauth_translate_activesync_ews_usr(ngx_pool_t *pool, ngx_str_t *src ngx_str_t *tgt); static ngx_flag_t ngx_field_from_zmauthtoken(ngx_log_t *log, ngx_pool_t *pool, ngx_str_t *authtoken, ngx_str_t *field, ngx_str_t *value); +static ngx_flag_t ngx_claimdata_from_jwttoken(ngx_log_t *log, ngx_pool_t *pool, + ngx_str_t *authtoken, ngx_str_t *field, ngx_str_t *value); +static ngx_flag_t ngx_get_authheader_bearer(ngx_log_t *log, ngx_pool_t *pool, + ngx_http_headers_in_t *headers_in, ngx_str_t *value); +static ngx_flag_t ngx_claimdata_from_payload(ngx_log_t *log, ngx_pool_t *pool, + ngx_str_t *payload, ngx_str_t *value); static ngx_flag_t ngx_get_cookie_value(ngx_log_t *log, ngx_table_elt_t **cookies, ngx_uint_t ncookies, ngx_str_t *name, ngx_str_t *value); static ngx_flag_t ngx_get_query_string_arg(ngx_log_t *log, ngx_str_t *args, ngx_str_t *name, ngx_str_t *value); + static ngx_str_t NGX_ZMAUTHTOKEN_ID = ngx_string("id"); static ngx_str_t NGX_ZMAUTHTOKEN_VER = ngx_string("version"); static ngx_str_t NGX_ZMAUTHTOKEN = ngx_string("ZM_AUTH_TOKEN"); @@ -89,6 +97,9 @@ static ngx_str_t NGX_ZMAUTHTOKEN_ADMIN = ngx_string("ZM_ADMIN_AUTH_TOKEN"); static ngx_str_t NGX_ZAUTHTOKEN = ngx_string("zauthtoken"); static ngx_str_t NGX_ACCESS_DENIED_ERRMSG = ngx_string("is not allowed on this domain"); +static ngx_str_t NGX_ZMJWT_CLAIM_ID = ngx_string("sub"); +static ngx_str_t NGX_ZJWT_QUERY = ngx_string("zjwt"); + static ngx_command_t ngx_http_upstream_zmauth_commands[] = { { ngx_string("upstream_fair_shm_size"), @@ -330,7 +341,6 @@ ngx_http_upstream_do_init_zmauth_peer(ngx_http_request_t *r, if (pool == NULL) { return NGX_ERROR; } - ctx = ngx_pcalloc(pool, sizeof(ngx_http_upstream_zmauth_ctx_t)); if (ctx == NULL) { ngx_destroy_pool(pool); @@ -338,7 +348,6 @@ ngx_http_upstream_do_init_zmauth_peer(ngx_http_request_t *r, } ngx_http_set_ctx(r, ctx, ngx_http_upstream_zmauth_module); - ctx->pool = pool; ctx->zmp = zmp; @@ -346,7 +355,6 @@ ngx_http_upstream_do_init_zmauth_peer(ngx_http_request_t *r, ngx_http_cleanup_t * cln = ngx_http_cleanup_add(r, 0); cln->handler = ngx_http_upstream_zmauth_cleanup; cln->data = r; - work = ngx_pcalloc(pool, sizeof(ngx_zm_lookup_work_t)); if(work == NULL) { ngx_destroy_pool(pool); @@ -363,7 +371,6 @@ ngx_http_upstream_do_init_zmauth_peer(ngx_http_request_t *r, work->alias_check_stat = ZM_ALIAS_NOT_CHECKED; work->on_success = zmauth_lookup_result_handler; work->on_failure = zmauth_lookup_result_handler; - schema = r->upstream->schema; if (schema.len == sizeof("http://") - 1 && ngx_strncmp(schema.data, (u_char *)"http://", sizeof("http://") - 1) == 0) { @@ -373,18 +380,16 @@ ngx_http_upstream_do_init_zmauth_peer(ngx_http_request_t *r, work->protocol = ZM_PROTO_HTTPS; } - if (zmp->zmroutetype == zmroutetype_authtoken) { + if (zmp->zmroutetype == zmroutetype_authtoken || zmp->zmroutetype == zmroutetype_jwttoken) { work->auth_method = ZM_AUTHMETH_ZIMBRAID; } else { work->auth_method = ZM_AUTHMETH_USERNAME; } - if (type == zmauth_admin_console) { work->isAdmin = 1; } ctx->work = work; ctx->connect = us->connect; - ngx_zm_lookup(work); return NGX_AGAIN; // return NGX_AGAIN to indicate this is an async peer init @@ -1106,10 +1111,10 @@ ngx_claimdata_from_jwttoken(ngx_log_t *log, ngx_pool_t *pool, return 0; } - int captures[30]; + int captures[(1 + rgc.captures) * 3]; ngx_int_t n; - n = ngx_regex_exec(rgc.regex, authtoken, captures, 30); + n = ngx_regex_exec(rgc.regex, authtoken, captures, (1 + rgc.captures) * 3); if (n >= 0) { /* string matches expression */ @@ -1120,13 +1125,19 @@ ngx_claimdata_from_jwttoken(ngx_log_t *log, ngx_pool_t *pool, ngx_log_debug1(NGX_LOG_DEBUG_ZIMBRA, log, 0, "zmauth: ngx_claimdata_from_jwttoken payload is %V",&encodedPayload); ngx_str_t urlDecodedPayload; + urlDecodedPayload.data = ngx_pnalloc(pool, ngx_base64_decoded_length(encodedPayload.len)); + if (urlDecodedPayload.data == NULL) + { + ngx_log_error(NGX_LOG_INFO, log, 0,"zmauth: ngx_claimdata_from_jwttoken memory allocation from pool failed"); + return NGX_ERROR; + } + if (ngx_decode_base64url(&urlDecodedPayload, &encodedPayload) != NGX_OK) { ngx_log_error(NGX_LOG_INFO, log, 0,"zmauth: ngx_claimdata_from_jwttoken base64url decoding failed"); return 0; } ngx_log_debug1(NGX_LOG_DEBUG_ZIMBRA, log, 0, "zmauth: ngx_claimdata_from_jwttoken base64url decoded payload is %V",&urlDecodedPayload); - int ret = ngx_claimdata_from_payload(log,pool,&urlDecodedPayload,value); return ret; } @@ -1134,7 +1145,7 @@ ngx_claimdata_from_jwttoken(ngx_log_t *log, ngx_pool_t *pool, { /* no match was found */ ngx_log_debug0(NGX_LOG_DEBUG_ZIMBRA, log, 0, "zmauth: ngx_claimdata_from_jwttoken no match found"); - return 0; + return 0; } else { @@ -1145,11 +1156,10 @@ ngx_claimdata_from_jwttoken(ngx_log_t *log, ngx_pool_t *pool, ngx_log_debug0 (NGX_LOG_DEBUG_HTTP, log, 0, "zmauth: ngx_claimdata_from_jwttoken exit"); - + return 0; } - /* extract sub data field from payload */ static ngx_flag_t ngx_claimdata_from_payload(ngx_log_t *log, ngx_pool_t *pool, @@ -1172,14 +1182,14 @@ ngx_claimdata_from_payload(ngx_log_t *log, ngx_pool_t *pool, if (ngx_regex_compile(&rgc) != NGX_OK) { - ngx_log_error(NGX_LOG_INFO, log, 0,"zmauth: ngx_claimdata_from_payload regex compilation failed.Error=====%s",errstr); + ngx_log_error(NGX_LOG_INFO, log, 0,"zmauth: ngx_claimdata_from_payload regex compilation failed.Error:%s",errstr); return 0; } - int captures[30]; + int captures[(1 + rgc.captures) * 3]; ngx_int_t n; - n = ngx_regex_exec(rgc.regex, payload, captures, 30); + n = ngx_regex_exec(rgc.regex, payload, captures, (1 + rgc.captures) * 3); if (n >= 0) { /* string matches expression */ @@ -1655,6 +1665,79 @@ zmauth_check_authtoken(ngx_http_request_t *r, ngx_str_t* field, return f; } + +/* examine request header and query parameter for jwt token and extract route if so */ +static ngx_flag_t +zmauth_check_jwttoken(ngx_http_request_t *r, ngx_str_t* field, + void **extra) { + ngx_pool_t *pool; + ngx_log_t *log; + ngx_str_t token, value, *pvalue; + ngx_flag_t f; + + pool = r->pool; + log = r->connection->log; + + ngx_log_debug0 (NGX_LOG_DEBUG_HTTP, log, 0, "zmauth:zmauth_check_jwttoken starts"); + + + //Bearer header extration starts + if (r->headers_in.authorization != NULL + && r->headers_in.authorization->value.data != NULL) + { + ngx_log_debug0 (NGX_LOG_DEBUG_HTTP, log, 0, "zmauth:zmauth_check_jwttoken authorization header exists"); + f = ngx_get_authheader_bearer(log, pool, &r->headers_in, &token); + } + //Bearer header extraction ends + + + if (!f) { + /* if not found, then look in the query string arg zjwt*/ + f = ngx_get_query_string_arg(log, &r->args, &NGX_ZJWT_QUERY, &token); + } + + if (f) { + ngx_log_debug1 (NGX_LOG_DEBUG_HTTP, log, 0, + "zmauth:zmauth_check_jwttoken found ZM_JWT_TOKEN:%V", + &token); + + f = ngx_claimdata_from_jwttoken(log, pool, &token, field, &value); + + if (f) { + ngx_log_debug2 (NGX_LOG_DEBUG_HTTP, log, 0, "zmauth: got %V:%V from ZM_AUTH_TOKEN", field, &value); + if (value.len > 0) { + pvalue = ngx_palloc(pool, sizeof(ngx_str_t)); + if (pvalue == NULL) { + f = 0; + } else { + pvalue->data = ngx_pstrdup(pool, &value); /* TODO: shallowcopy? */ + if (pvalue->data == NULL) { + f = 0; + } else { + pvalue->len = value.len; + *((ngx_str_t**) extra) = pvalue; + } + } + } else { + f = 0; + } + } else { + ngx_log_debug1 (NGX_LOG_DEBUG_HTTP, log, 0, + "zmauth: no %V in ZM_AUTH_TOKEN", field); + } + + } else { + ngx_log_debug1 (NGX_LOG_DEBUG_HTTP, log, 0, + "zmauth: no ZM_AUTH_TOKEN", + &token); + } + + ngx_log_debug0 (NGX_LOG_DEBUG_HTTP, log, 0, "zmauth:zmauth_check_jwttoken ends"); + return f; +} + + + /* examine request cookies for ZM_ADMIN_AUTH_TOKEN and extract route if so */ static ngx_flag_t zmauth_check_admin_authtoken(ngx_http_request_t *r, ngx_str_t* field, @@ -1741,6 +1824,10 @@ zmauth_check_uri(ngx_http_request_t *r, void **extra) { rtype = zmroutetype_authtoken; ngx_log_debug0(NGX_LOG_DEBUG_HTTP,r->connection->log,0, "zmauth: routing by ZM_AUTH_TOKEN"); + } else if (zmauth_check_jwttoken(r, &NGX_ZMJWT_CLAIM_ID, extra)) { + rtype = zmroutetype_jwttoken; + ngx_log_debug0(NGX_LOG_DEBUG_HTTP,r->connection->log,0, + "zmauth: routing by ZM_JWT_TOKEN"); } else { ngx_log_debug0(NGX_LOG_DEBUG_HTTP,r->connection->log,0, "zmauth: routing by iphash"); diff --git a/thirdparty/nginx/nginx-1.7.1-zimbra/src/http/modules/ngx_http_upstream_zmauth_module.h b/thirdparty/nginx/nginx-1.7.1-zimbra/src/http/modules/ngx_http_upstream_zmauth_module.h index b641aac32..325d64c76 100644 --- a/thirdparty/nginx/nginx-1.7.1-zimbra/src/http/modules/ngx_http_upstream_zmauth_module.h +++ b/thirdparty/nginx/nginx-1.7.1-zimbra/src/http/modules/ngx_http_upstream_zmauth_module.h @@ -22,7 +22,8 @@ typedef enum { zmroutetype_rest, zmroutetype_activesync, zmroutetype_caldav, - zmroutetype_ews + zmroutetype_ews, + zmroutetype_jwttoken, } zmroutetype_t; typedef struct { From a1d7e0a06f08fb81f0a1032f176ef951afc1df38 Mon Sep 17 00:00:00 2001 From: Shashi Santosh Date: Thu, 19 Apr 2018 15:04:35 +0530 Subject: [PATCH 05/15] ZCS-4021:Upgrading regex to non greedy --- .../modules/ngx_http_upstream_zmauth_module.c | 112 ++++++++---------- .../nginx/zimbra-nginx/rpm/SPECS/nginx.spec | 2 + 2 files changed, 54 insertions(+), 60 deletions(-) diff --git a/thirdparty/nginx/nginx-1.7.1-zimbra/src/http/modules/ngx_http_upstream_zmauth_module.c b/thirdparty/nginx/nginx-1.7.1-zimbra/src/http/modules/ngx_http_upstream_zmauth_module.c index 8addfb73f..ae5f7bb13 100644 --- a/thirdparty/nginx/nginx-1.7.1-zimbra/src/http/modules/ngx_http_upstream_zmauth_module.c +++ b/thirdparty/nginx/nginx-1.7.1-zimbra/src/http/modules/ngx_http_upstream_zmauth_module.c @@ -1051,8 +1051,7 @@ ngx_field_from_zmauthtoken(ngx_log_t *log, ngx_pool_t *pool, /* extract the jwt token from Header*/ static ngx_flag_t ngx_get_authheader_bearer(ngx_log_t *log, ngx_pool_t *pool, - ngx_http_headers_in_t *headers_in, ngx_str_t *value) -{ + ngx_http_headers_in_t *headers_in, ngx_str_t *value) { ngx_log_debug0 (NGX_LOG_DEBUG_HTTP, log, 0, "zmauth: ngx_get_authheader_bearer entered"); @@ -1061,19 +1060,21 @@ ngx_get_authheader_bearer(ngx_log_t *log, ngx_pool_t *pool, authValue = headers_in->authorization->value; - ngx_log_debug1 (NGX_LOG_DEBUG_HTTP, log, 0, "zmauth: ngx_get_authheader_bearer token:%V",&authValue); + ngx_log_debug1 (NGX_LOG_DEBUG_HTTP, log, 0, + "zmauth: ngx_get_authheader_bearer token:%V", &authValue); if (authValue.len >= sizeof("Bearer ") - 1 && ngx_memcmp(authValue.data,"Bearer ",sizeof("Bearer ")-1) - == 0) - { - ngx_log_debug0 (NGX_LOG_DEBUG_HTTP, log, 0, "zmauth: ngx_get_authheader_bearer Bearer token found"); + == 0) { + ngx_log_debug0 (NGX_LOG_DEBUG_HTTP, log, 0, + "zmauth: ngx_get_authheader_bearer Bearer token found"); token = authValue; token.data += (sizeof("Bearer ") - 1); token.len -= (sizeof("Bearer ") - 1); - ngx_log_debug1 (NGX_LOG_DEBUG_HTTP, log, 0, "zmauth: ngx_get_authheader_bearer Bearer token found:%V",&token); + ngx_log_debug1 (NGX_LOG_DEBUG_HTTP, log, 0, + "zmauth: ngx_get_authheader_bearer Bearer token found:%V", &token); *value = token; return 1; @@ -1089,7 +1090,6 @@ ngx_get_authheader_bearer(ngx_log_t *log, ngx_pool_t *pool, static ngx_flag_t ngx_claimdata_from_jwttoken(ngx_log_t *log, ngx_pool_t *pool, ngx_str_t *authtoken, ngx_str_t *field, ngx_str_t *value) { - ngx_flag_t f; ngx_log_debug0 (NGX_LOG_DEBUG_HTTP, log, 0, "zmauth: ngx_claimdata_from_jwttoken entered"); @@ -1103,11 +1103,9 @@ ngx_claimdata_from_jwttoken(ngx_log_t *log, ngx_pool_t *pool, rgc.pool = pool; rgc.err.len = NGX_MAX_CONF_ERRSTR; rgc.err.data = errstr; - - - if (ngx_regex_compile(&rgc) != NGX_OK) - { - ngx_log_error(NGX_LOG_INFO, log, 0,"zmauth: ngx_claimdata_from_jwttoken regex compilation failed.Error=====%s",errstr); + if (ngx_regex_compile(&rgc) != NGX_OK) { + ngx_log_error(NGX_LOG_INFO, log, 0, + "zmauth: ngx_claimdata_from_jwttoken regex compilation failed. Error:%s", errstr); return 0; } @@ -1115,42 +1113,44 @@ ngx_claimdata_from_jwttoken(ngx_log_t *log, ngx_pool_t *pool, ngx_int_t n; n = ngx_regex_exec(rgc.regex, authtoken, captures, (1 + rgc.captures) * 3); - if (n >= 0) - { + if (n >= 0) { /* string matches expression */ - ngx_log_debug0(NGX_LOG_DEBUG_ZIMBRA, log, 0, "zmauth: ngx_claimdata_from_jwttoken match found"); + ngx_log_debug0(NGX_LOG_DEBUG_ZIMBRA, log, 0, + "zmauth: ngx_claimdata_from_jwttoken match found"); ngx_str_t encodedPayload; encodedPayload.data = authtoken->data + captures[0]; encodedPayload.len = captures[1] - captures[0]; - ngx_log_debug1(NGX_LOG_DEBUG_ZIMBRA, log, 0, "zmauth: ngx_claimdata_from_jwttoken payload is %V",&encodedPayload); + ngx_log_debug1(NGX_LOG_DEBUG_ZIMBRA, log, 0, + "zmauth: ngx_claimdata_from_jwttoken payload is %V", &encodedPayload); ngx_str_t urlDecodedPayload; urlDecodedPayload.data = ngx_pnalloc(pool, ngx_base64_decoded_length(encodedPayload.len)); - if (urlDecodedPayload.data == NULL) - { - ngx_log_error(NGX_LOG_INFO, log, 0,"zmauth: ngx_claimdata_from_jwttoken memory allocation from pool failed"); + if (urlDecodedPayload.data == NULL) { + ngx_log_error(NGX_LOG_INFO, log, 0, + "zmauth: ngx_claimdata_from_jwttoken memory allocation from pool failed"); return NGX_ERROR; } - if (ngx_decode_base64url(&urlDecodedPayload, &encodedPayload) != NGX_OK) - { - ngx_log_error(NGX_LOG_INFO, log, 0,"zmauth: ngx_claimdata_from_jwttoken base64url decoding failed"); + if (ngx_decode_base64url(&urlDecodedPayload, &encodedPayload) != NGX_OK) { + ngx_log_error(NGX_LOG_INFO, log, 0, + "zmauth: ngx_claimdata_from_jwttoken base64url decoding failed"); return 0; } - ngx_log_debug1(NGX_LOG_DEBUG_ZIMBRA, log, 0, "zmauth: ngx_claimdata_from_jwttoken base64url decoded payload is %V",&urlDecodedPayload); + ngx_log_debug1(NGX_LOG_DEBUG_ZIMBRA, log, 0, + "zmauth: ngx_claimdata_from_jwttoken base64url decoded payload is %V", &urlDecodedPayload); int ret = ngx_claimdata_from_payload(log,pool,&urlDecodedPayload,value); return ret; } - else if (n == NGX_REGEX_NO_MATCHED) - { + else if (n == NGX_REGEX_NO_MATCHED) { /* no match was found */ - ngx_log_debug0(NGX_LOG_DEBUG_ZIMBRA, log, 0, "zmauth: ngx_claimdata_from_jwttoken no match found"); + ngx_log_debug0(NGX_LOG_DEBUG_ZIMBRA, log, 0, + "zmauth: ngx_claimdata_from_jwttoken no match found"); return 0; } - else - { + else { /* some error */ - ngx_log_debug0(NGX_LOG_DEBUG_ZIMBRA, log, 0, "zmauth: ngx_claimdata_from_jwttoken no match found due to some error."); + ngx_log_debug1(NGX_LOG_DEBUG_ZIMBRA, log, 0, + "zmauth: ngx_claimdata_from_jwttoken failed: %i", n); return 0; } @@ -1164,12 +1164,11 @@ ngx_claimdata_from_jwttoken(ngx_log_t *log, ngx_pool_t *pool, static ngx_flag_t ngx_claimdata_from_payload(ngx_log_t *log, ngx_pool_t *pool, ngx_str_t *payload, ngx_str_t *value) { - ngx_flag_t f; ngx_log_debug0 (NGX_LOG_DEBUG_HTTP, log, 0, "zmauth: ngx_claimdata_from_payload entered"); - ngx_str_t pattern = ngx_string("\"sub\":[\s]*\"(.*)\"[\s]*"); + ngx_str_t pattern = ngx_string("\"sub\":[\\s]*\"(.*?)\"[\\s]*"); ngx_regex_compile_t rgc; u_char errstr[NGX_MAX_CONF_ERRSTR]; ngx_memzero(&rgc, sizeof(ngx_regex_compile_t)); @@ -1178,11 +1177,9 @@ ngx_claimdata_from_payload(ngx_log_t *log, ngx_pool_t *pool, rgc.pool = pool; rgc.err.len = NGX_MAX_CONF_ERRSTR; rgc.err.data = errstr; - - - if (ngx_regex_compile(&rgc) != NGX_OK) - { - ngx_log_error(NGX_LOG_INFO, log, 0,"zmauth: ngx_claimdata_from_payload regex compilation failed.Error:%s",errstr); + if (ngx_regex_compile(&rgc) != NGX_OK) { + ngx_log_error(NGX_LOG_INFO, log, 0, + "zmauth: ngx_claimdata_from_payload regex compilation failed. Error:%s", errstr); return 0; } @@ -1190,27 +1187,28 @@ ngx_claimdata_from_payload(ngx_log_t *log, ngx_pool_t *pool, ngx_int_t n; n = ngx_regex_exec(rgc.regex, payload, captures, (1 + rgc.captures) * 3); - if (n >= 0) - { + if (n >= 0) { /* string matches expression */ - ngx_log_debug0(NGX_LOG_DEBUG_ZIMBRA, log, 0, "zmauth: ngx_claimdata_from_payload match found"); + ngx_log_debug0(NGX_LOG_DEBUG_ZIMBRA, log, 0, + "zmauth: ngx_claimdata_from_payload match found"); ngx_str_t subData; subData.data = payload->data + captures[2]; subData.len = captures[3] - captures[2]; - ngx_log_debug1(NGX_LOG_DEBUG_ZIMBRA, log, 0, "zmauth: ngx_claimdata_from_payload subData is %V",&subData); + ngx_log_debug1(NGX_LOG_DEBUG_ZIMBRA, log, 0, + "zmauth: ngx_claimdata_from_payload subData is %V", &subData); *value = subData; return 1; } - else if (n == NGX_REGEX_NO_MATCHED) - { + else if (n == NGX_REGEX_NO_MATCHED) { /* no match was found */ - ngx_log_debug0(NGX_LOG_DEBUG_ZIMBRA, log, 0, "zmauth: ngx_claimdata_from_payload no match found"); + ngx_log_debug0(NGX_LOG_DEBUG_ZIMBRA, log, 0, + "zmauth: ngx_claimdata_from_payload no match found"); return 0; } - else - { + else { /* some error */ - ngx_log_debug0(NGX_LOG_DEBUG_ZIMBRA, log, 0, "zmauth: ngx_claimdata_from_payload no match found due to some error."); + ngx_log_debug1(NGX_LOG_DEBUG_ZIMBRA, log, 0, + "zmauth: ngx_claimdata_from_jwttoken failed: %i", n); return 0; } @@ -1220,8 +1218,6 @@ ngx_claimdata_from_payload(ngx_log_t *log, ngx_pool_t *pool, return 0; } - - static void * ngx_http_upstream_zmauth_create_srv_conf(ngx_conf_t *cf) { ngx_http_upstream_zmauth_srv_conf_t *zscf; @@ -1677,20 +1673,16 @@ zmauth_check_jwttoken(ngx_http_request_t *r, ngx_str_t* field, pool = r->pool; log = r->connection->log; - ngx_log_debug0 (NGX_LOG_DEBUG_HTTP, log, 0, "zmauth:zmauth_check_jwttoken starts"); - - - //Bearer header extration starts + //Bearer header extraction starts if (r->headers_in.authorization != NULL - && r->headers_in.authorization->value.data != NULL) - { - ngx_log_debug0 (NGX_LOG_DEBUG_HTTP, log, 0, "zmauth:zmauth_check_jwttoken authorization header exists"); + && r->headers_in.authorization->value.data != NULL) { + ngx_log_debug0 (NGX_LOG_DEBUG_HTTP, log, 0, + "zmauth:zmauth_check_jwttoken authorization header exists"); f = ngx_get_authheader_bearer(log, pool, &r->headers_in, &token); } //Bearer header extraction ends - if (!f) { /* if not found, then look in the query string arg zjwt*/ f = ngx_get_query_string_arg(log, &r->args, &NGX_ZJWT_QUERY, &token); @@ -1698,13 +1690,13 @@ zmauth_check_jwttoken(ngx_http_request_t *r, ngx_str_t* field, if (f) { ngx_log_debug1 (NGX_LOG_DEBUG_HTTP, log, 0, - "zmauth:zmauth_check_jwttoken found ZM_JWT_TOKEN:%V", - &token); + "zmauth:zmauth_check_jwttoken found ZM_JWT_TOKEN:%V", &token); f = ngx_claimdata_from_jwttoken(log, pool, &token, field, &value); if (f) { - ngx_log_debug2 (NGX_LOG_DEBUG_HTTP, log, 0, "zmauth: got %V:%V from ZM_AUTH_TOKEN", field, &value); + ngx_log_debug2 (NGX_LOG_DEBUG_HTTP, log, 0, + "zmauth: got %V:%V from ZM_AUTH_TOKEN", field, &value); if (value.len > 0) { pvalue = ngx_palloc(pool, sizeof(ngx_str_t)); if (pvalue == NULL) { diff --git a/thirdparty/nginx/zimbra-nginx/rpm/SPECS/nginx.spec b/thirdparty/nginx/zimbra-nginx/rpm/SPECS/nginx.spec index babbe546d..f87436a96 100644 --- a/thirdparty/nginx/zimbra-nginx/rpm/SPECS/nginx.spec +++ b/thirdparty/nginx/zimbra-nginx/rpm/SPECS/nginx.spec @@ -19,6 +19,8 @@ The Zimbra nginx build %changelog * Fri Aug 24 2018 Zimbra Packaging Services - VERSION-1zimbra8.7b8ZAPPEND - Patch for nginx Bug 107566 +* Wed Apr 18 2018 Zimbra Packaging Services - VERSION-1zimbra8.7b8ZAPPEND +- ZCS-4021 for JWT support. * Wed May 10 2017 Zimbra Packaging Services - VERSION-1zimbra8.7b7ZAPPEND - Patch for nginx Bug 107438 (and 106918, 106876) - Patch for nginx Bug 106948. From 8e204fd7740860c730f02fa13215be304ce79047 Mon Sep 17 00:00:00 2001 From: Shashi Santosh Date: Mon, 27 Aug 2018 17:05:04 +0530 Subject: [PATCH 06/15] ZCS-4021: Updated changelog and version --- thirdparty/nginx/zimbra-nginx/debian/changelog | 6 ++++++ thirdparty/nginx/zimbra-nginx/rpm/SPECS/nginx.spec | 6 +++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/thirdparty/nginx/zimbra-nginx/debian/changelog b/thirdparty/nginx/zimbra-nginx/debian/changelog index 095c7a298..1d0cba274 100644 --- a/thirdparty/nginx/zimbra-nginx/debian/changelog +++ b/thirdparty/nginx/zimbra-nginx/debian/changelog @@ -1,3 +1,9 @@ +zimbra-nginx (VERSION-1zimbra8.7b9ZAPPEND) unstable; urgency=medium + + * ZCS-4021 for JWT support. + + -- Zimbra Packaging Services Wed, 30 Aug 2018 22:48:22 +0030 + zimbra-nginx (VERSION-1zimbra8.7b8ZAPPEND) unstable; urgency=medium * Patch for nginx Bug 107566 diff --git a/thirdparty/nginx/zimbra-nginx/rpm/SPECS/nginx.spec b/thirdparty/nginx/zimbra-nginx/rpm/SPECS/nginx.spec index f87436a96..2f30c2817 100644 --- a/thirdparty/nginx/zimbra-nginx/rpm/SPECS/nginx.spec +++ b/thirdparty/nginx/zimbra-nginx/rpm/SPECS/nginx.spec @@ -1,7 +1,7 @@ Summary: Zimbra's nginx build Name: zimbra-nginx Version: VERSION -Release: 1zimbra8.7b8ZAPPEND +Release: 1zimbra8.7b9ZAPPEND License: MIT Source: %{name}-%{version}.tar.gz BuildRequires: pcre-devel, zlib-devel @@ -17,10 +17,10 @@ URL: http://nginx.org The Zimbra nginx build %changelog +* Thu Aug 30 2018 Zimbra Packaging Services - VERSION-1zimbra8.7b9ZAPPEND +- ZCS-4021 for JWT support. * Fri Aug 24 2018 Zimbra Packaging Services - VERSION-1zimbra8.7b8ZAPPEND - Patch for nginx Bug 107566 -* Wed Apr 18 2018 Zimbra Packaging Services - VERSION-1zimbra8.7b8ZAPPEND -- ZCS-4021 for JWT support. * Wed May 10 2017 Zimbra Packaging Services - VERSION-1zimbra8.7b7ZAPPEND - Patch for nginx Bug 107438 (and 106918, 106876) - Patch for nginx Bug 106948. From 972e17672ebfdb6de5bdc7c30a95d6a8d601eb3a Mon Sep 17 00:00:00 2001 From: Amit Srivastava Date: Wed, 5 Sep 2018 17:57:19 +0530 Subject: [PATCH 07/15] ZCS-5907: proxy-component: version update 1.0.2 --- .../zimbra-proxy-components/debian/changelog | 3 +++ .../proxy-components/zimbra-proxy-components/debian/control | 2 +- .../zimbra-proxy-components/rpm/SPECS/proxy-components.spec | 6 ++++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/zimbra/proxy-components/zimbra-proxy-components/debian/changelog b/zimbra/proxy-components/zimbra-proxy-components/debian/changelog index 2e54882a5..d88e876b4 100644 --- a/zimbra/proxy-components/zimbra-proxy-components/debian/changelog +++ b/zimbra/proxy-components/zimbra-proxy-components/debian/changelog @@ -1,3 +1,6 @@ +zimbra-proxy-components (1.0.2-ITERATIONZAPPEND) unstable; urgency=low + + * Updated zimbra-nginx package zimbra-proxy-components (1.0.1-ITERATIONZAPPEND) unstable; urgency=low * Updated zimbra-nginx package diff --git a/zimbra/proxy-components/zimbra-proxy-components/debian/control b/zimbra/proxy-components/zimbra-proxy-components/debian/control index c39add303..a6f18178b 100644 --- a/zimbra/proxy-components/zimbra-proxy-components/debian/control +++ b/zimbra/proxy-components/zimbra-proxy-components/debian/control @@ -8,7 +8,7 @@ Standards-Version: 3.9.5 Package: zimbra-proxy-components Architecture: all Depends: ${misc:Depends}, ${perl:Depends}, ${shlibs:Depends}, - zimbra-proxy-base, zimbra-nginx (>= 1.7.1-1zimbra8.7b7ZAPPEND) + zimbra-proxy-base, zimbra-nginx (>= 1.7.1-1zimbra8.7b9ZAPPEND) Description: Zimbra components for proxy package Zimbra proxy components pulls in all the packages used by zimbra-proxy diff --git a/zimbra/proxy-components/zimbra-proxy-components/rpm/SPECS/proxy-components.spec b/zimbra/proxy-components/zimbra-proxy-components/rpm/SPECS/proxy-components.spec index 2ed63ddc2..03c67307c 100644 --- a/zimbra/proxy-components/zimbra-proxy-components/rpm/SPECS/proxy-components.spec +++ b/zimbra/proxy-components/zimbra-proxy-components/rpm/SPECS/proxy-components.spec @@ -1,9 +1,9 @@ Summary: Zimbra components for proxy package Name: zimbra-proxy-components -Version: 1.0.1 +Version: 1.0.2 Release: ITERATIONZAPPEND License: GPL-2 -Requires: zimbra-proxy-base, zimbra-nginx >= 1.7.1-1zimbra8.7b7ZAPPEND +Requires: zimbra-proxy-base, zimbra-nginx >= 1.7.1-1zimbra8.7b9ZAPPEND Packager: Zimbra Packaging Services Group: Development/Languages AutoReqProv: no @@ -11,6 +11,8 @@ AutoReqProv: no %define debug_package %{nil} %changelog +* Tue Sep 5 2018 Zimbra Packaging Services - 1.0.2 +- Updated zimbra-nginx package * Tue Jul 18 2017 Zimbra Packaging Services - 1.0.1 - Updated zimbra-nginx package * Wed Sep 09 2015 Zimbra Packaging Services - 1.0.0 From 0b4405468133b641bfeb5c9f62b35624b76f574f Mon Sep 17 00:00:00 2001 From: Amit Srivastava Date: Wed, 5 Sep 2018 18:18:57 +0530 Subject: [PATCH 08/15] ZCS-5907: proxy-component: version update 1.0.2 --- .../proxy-components/zimbra-proxy-components/debian/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zimbra/proxy-components/zimbra-proxy-components/debian/changelog b/zimbra/proxy-components/zimbra-proxy-components/debian/changelog index d88e876b4..56e9b27c2 100644 --- a/zimbra/proxy-components/zimbra-proxy-components/debian/changelog +++ b/zimbra/proxy-components/zimbra-proxy-components/debian/changelog @@ -1,6 +1,8 @@ zimbra-proxy-components (1.0.2-ITERATIONZAPPEND) unstable; urgency=low * Updated zimbra-nginx package + + -- Zimbra Packaging Services Tue, 18 Jul 2017 00:00:00 +0000 zimbra-proxy-components (1.0.1-ITERATIONZAPPEND) unstable; urgency=low * Updated zimbra-nginx package From c10c2b374937f18592df3f0f68fb37ee3ad66247 Mon Sep 17 00:00:00 2001 From: Quanah Gibson-Mount Date: Mon, 10 Sep 2018 19:40:15 +0000 Subject: [PATCH 09/15] Multival updates * Update multival patch for new options * Increase default LMDB max keysize * Add minor patch for issue fixed in upcoming 2.4.47 release --- .../openldap/patches/index-delete.patch | 15 + .../openldap/patches/liblmdb-keysize.patch | 11 + thirdparty/openldap/patches/multival.patch | 603 ++++++++++++++---- .../openldap/zimbra-openldap/debian/changelog | 7 + .../zimbra-openldap/debian/patches/series | 2 + .../zimbra-openldap/rpm/SPECS/openldap.spec | 9 +- 6 files changed, 519 insertions(+), 128 deletions(-) create mode 100644 thirdparty/openldap/patches/index-delete.patch create mode 100644 thirdparty/openldap/patches/liblmdb-keysize.patch diff --git a/thirdparty/openldap/patches/index-delete.patch b/thirdparty/openldap/patches/index-delete.patch new file mode 100644 index 000000000..a14b19574 --- /dev/null +++ b/thirdparty/openldap/patches/index-delete.patch @@ -0,0 +1,15 @@ +--- openldap-2.4.46/servers/slapd/back-mdb/config.c.orig 2018-08-31 21:34:31.111309728 +0000 ++++ openldap-2.4.46/servers/slapd/back-mdb/config.c 2018-08-31 21:22:48.499227430 +0000 +@@ -423,10 +435,11 @@ + if ( c->valx == -1 ) { + int i; + +- /* delete all (FIXME) */ ++ /* delete all */ + for ( i = 0; i < mdb->mi_nattrs; i++ ) { + mdb->mi_attrs[i]->ai_indexmask |= MDB_INDEX_DELETING; + } ++ mdb->mi_defaultmask = 0; + mdb->mi_flags |= MDB_DEL_INDEX; + c->cleanup = mdb_cf_cleanup; + diff --git a/thirdparty/openldap/patches/liblmdb-keysize.patch b/thirdparty/openldap/patches/liblmdb-keysize.patch new file mode 100644 index 000000000..673b3aea7 --- /dev/null +++ b/thirdparty/openldap/patches/liblmdb-keysize.patch @@ -0,0 +1,11 @@ +--- openldap-2.4.46/libraries/liblmdb/mdb.c.orig 2018-09-07 17:18:22.384349450 +0000 ++++ openldap-2.4.46/libraries/liblmdb/mdb.c 2018-09-07 17:19:31.115615450 +0000 +@@ -555,7 +555,7 @@ + * #MDB_DUPSORT data items must fit on a node in a regular page. + */ + #ifndef MDB_MAXKEYSIZE +-#define MDB_MAXKEYSIZE ((MDB_DEVEL) ? 0 : 511) ++#define MDB_MAXKEYSIZE ((MDB_DEVEL) ? 0 : 1533) + #endif + + /** The maximum size of a key we can write to the environment. */ diff --git a/thirdparty/openldap/patches/multival.patch b/thirdparty/openldap/patches/multival.patch index 062e3c7fa..9f5be348c 100644 --- a/thirdparty/openldap/patches/multival.patch +++ b/thirdparty/openldap/patches/multival.patch @@ -1,47 +1,281 @@ -commit cd31731d5d9d0226491b56f868073bcc588f063a -Author: Howard Chu -Date: Sun Jan 31 15:35:11 2016 +0000 - - Large multivalued attr support - - Store attrs with a large number of values separately from the - main entry blob. Note - we need support for large DUPSORT values - for this to be generally usable. - -diff --git a/doc/man/man5/slapd-mdb.5 b/doc/man/man5/slapd-mdb.5 -index 002ecf9..05af178 100644 ---- a/doc/man/man5/slapd-mdb.5 -+++ b/doc/man/man5/slapd-mdb.5 -@@ -162,6 +162,24 @@ Specify the file protection mode that newly created database +--- openldap-2.4.46/doc/man/man5/slapd-mdb.5.orig 2018-08-31 21:25:36.425302202 +0000 ++++ openldap-2.4.46/doc/man/man5/slapd-mdb.5 2018-08-31 21:20:15.245014918 +0000 +@@ -162,6 +162,23 @@ files should have. The default is 0600. .TP -+.BI multival_hi \ -+Specify the number of values above which a multivalued attribute is ++\fBmultival \fR{\fI\fR|\fBdefault\fR} \fI\fR,\fI ++Specify the number of values for which a multivalued attribute is +stored in a separate table. Normally entries are stored as a single +blob inside the database. When an entry gets very large or contains +attributes with a very large number of values, modifications on that +entry may get very slow. Splitting the large attributes out to a separate +table can improve the performance of modification operations. -+The default is UINT_MAX, which keeps all attributes in the main blob. -+.TP -+.BI multival_lo \ -+Specify the number of values below which a multivalued attribute -+that was stored in a separate table is moved back into the main -+entry blob. If a modification deletes enough values to bring an -+attribute below this threshold, its values will be removed from the -+separate table and merged back into the main entry blob. -+The default is UINT_MAX, which keeps all attributes in -+the main blob. ++The threshold is specified as a pair of integers. If the number of ++values exceeds the hi threshold the values will be split out. If ++a modification deletes enough values to bring an attribute below ++the lo threshold the values will be removed from the separate ++table and merged back into the main entry blob. ++The threshold can be set for a specific list of attributes, or ++the default can be configured for all other attributes. ++The default value for both hi and lo thresholds is UINT_MAX, which keeps ++all attributes in the main blob. +.TP .BI rtxnsize \ Specify the maximum number of entries to process in a single read transaction when executing a large search. Long-lived read transactions -diff --git a/servers/slapd/back-mdb/back-mdb.h b/servers/slapd/back-mdb/back-mdb.h -index b271069..f453a74 100644 ---- a/servers/slapd/back-mdb/back-mdb.h -+++ b/servers/slapd/back-mdb/back-mdb.h -@@ -32,7 +32,8 @@ LDAP_BEGIN_DECL +--- openldap-2.4.46/servers/slapd/slap.h.orig 2018-08-31 21:25:51.085135455 +0000 ++++ openldap-2.4.46/servers/slapd/slap.h 2018-08-31 21:20:08.589093362 +0000 +@@ -1163,10 +1163,11 @@ + #define SLAP_ATTR_DONT_FREE_DATA 0x4U + #define SLAP_ATTR_DONT_FREE_VALS 0x8U + #define SLAP_ATTR_SORTED_VALS 0x10U /* values are sorted */ ++#define SLAP_ATTR_BIG_MULTI 0x20U /* for backends */ + + /* These flags persist across an attr_dup() */ + #define SLAP_ATTR_PERSISTENT_FLAGS \ +- SLAP_ATTR_SORTED_VALS ++ (SLAP_ATTR_SORTED_VALS|SLAP_ATTR_BIG_MULTI) + + Attribute *a_next; + #ifdef LDAP_COMP_MATCH +--- openldap-2.4.46/servers/slapd/back-mdb/attr.c.orig 2018-08-31 21:35:03.686948796 +0000 ++++ openldap-2.4.46/servers/slapd/back-mdb/attr.c 2018-08-31 21:22:45.599260954 +0000 +@@ -122,6 +122,8 @@ + for ( i=0; imi_nattrs; i++ ) { + if ( mdb->mi_attrs[i]->ai_dbi ) /* already open */ + continue; ++ if ( !( mdb->mi_attrs[i]->ai_indexmask || mdb->mi_attrs[i]->ai_newmask )) /* not an index record */ ++ continue; + rc = mdb_dbi_open( txn, mdb->mi_attrs[i]->ai_desc->ad_type->sat_cname.bv_val, + flags, &mdb->mi_attrs[i]->ai_dbi ); + if ( rc ) { +@@ -382,6 +384,8 @@ + a->ai_root = NULL; + a->ai_desc = ad; + a->ai_dbi = 0; ++ a->ai_multi_hi = UINT_MAX; ++ a->ai_multi_lo = UINT_MAX; + + if ( mdb->mi_flags & MDB_IS_OPEN ) { + a->ai_indexmask = 0; +@@ -419,8 +423,16 @@ + #endif + rc = ainfo_insert( mdb, a ); + if( rc ) { ++ AttrInfo *b = mdb_attr_mask( mdb, ad ); ++ /* If this is just a multival record, reuse it for index info */ ++ if ( !( b->ai_indexmask || b->ai_newmask ) && b->ai_multi_lo < UINT_MAX ) { ++ b->ai_indexmask = a->ai_indexmask; ++ b->ai_newmask = a->ai_newmask; ++ ch_free( a ); ++ rc = 0; ++ continue; ++ } + if ( mdb->mi_flags & MDB_IS_OPEN ) { +- AttrInfo *b = mdb_attr_mask( mdb, ad ); + /* If there is already an index defined for this attribute + * it must be replaced. Otherwise we end up with multiple + * olcIndex values for the same attribute */ +@@ -490,7 +502,169 @@ + mdb_attr_index_unparser( &aidef, bva ); + } + for ( i=0; imi_nattrs; i++ ) +- mdb_attr_index_unparser( mdb->mi_attrs[i], bva ); ++ if ( mdb->mi_attrs[i]->ai_indexmask ) ++ mdb_attr_index_unparser( mdb->mi_attrs[i], bva ); ++} ++ ++int ++mdb_attr_multi_config( ++ struct mdb_info *mdb, ++ const char *fname, ++ int lineno, ++ int argc, ++ char **argv, ++ struct config_reply_s *c_reply) ++{ ++ int rc = 0; ++ int i; ++ unsigned hi,lo; ++ char **attrs, *next, *s; ++ ++ attrs = ldap_str2charray( argv[0], "," ); ++ ++ if( attrs == NULL ) { ++ fprintf( stderr, "%s: line %d: " ++ "no attributes specified: %s\n", ++ fname, lineno, argv[0] ); ++ return LDAP_PARAM_ERROR; ++ } ++ ++ hi = strtoul( argv[1], &next, 10 ); ++ if ( next == argv[1] || next[0] != ',' ) ++ goto badval; ++ s = next+1; ++ lo = strtoul( s, &next, 10 ); ++ if ( next == s || next[0] != '\0' ) ++ goto badval; ++ ++ if ( lo > hi ) { ++badval: ++ snprintf(c_reply->msg, sizeof(c_reply->msg), ++ "invalid hi/lo thresholds" ); ++ fprintf( stderr, "%s: line %d: %s\n", ++ fname, lineno, c_reply->msg ); ++ return LDAP_PARAM_ERROR; ++ } ++ ++ for ( i = 0; attrs[i] != NULL; i++ ) { ++ AttrInfo *a; ++ AttributeDescription *ad; ++ const char *text; ++ ++ if( strcasecmp( attrs[i], "default" ) == 0 ) { ++ mdb->mi_multi_hi = hi; ++ mdb->mi_multi_lo = lo; ++ continue; ++ } ++ ++ ad = NULL; ++ rc = slap_str2ad( attrs[i], &ad, &text ); ++ ++ if( rc != LDAP_SUCCESS ) { ++ if ( c_reply ) ++ { ++ snprintf(c_reply->msg, sizeof(c_reply->msg), ++ "multival attribute \"%s\" undefined", ++ attrs[i] ); ++ ++ fprintf( stderr, "%s: line %d: %s\n", ++ fname, lineno, c_reply->msg ); ++ } ++fail: ++ goto done; ++ } ++ ++ a = (AttrInfo *) ch_calloc( 1, sizeof(AttrInfo) ); ++ ++ a->ai_desc = ad; ++ a->ai_multi_hi = hi; ++ a->ai_multi_lo = lo; ++ ++ rc = ainfo_insert( mdb, a ); ++ if( rc ) { ++ AttrInfo *b = mdb_attr_mask( mdb, ad ); ++ /* If this is just an index record, reuse it for multival info */ ++ if ( b->ai_multi_lo == UINT_MAX ) { ++ b->ai_multi_hi = a->ai_multi_hi; ++ b->ai_multi_lo = a->ai_multi_lo; ++ ch_free( a ); ++ rc = 0; ++ continue; ++ } ++ if (c_reply) { ++ snprintf(c_reply->msg, sizeof(c_reply->msg), ++ "duplicate multival definition for attr \"%s\"", ++ attrs[i] ); ++ fprintf( stderr, "%s: line %d: %s\n", ++ fname, lineno, c_reply->msg ); ++ } ++ ++ rc = LDAP_PARAM_ERROR; ++ goto done; ++ } ++ } ++ ++done: ++ ldap_charray_free( attrs ); ++ ++ return rc; ++} ++ ++static int ++mdb_attr_multi_unparser( void *v1, void *v2 ) ++{ ++ AttrInfo *ai = v1; ++ BerVarray *bva = v2; ++ struct berval bv; ++ char digbuf[sizeof("4294967296,4294967296")]; ++ char *ptr; ++ ++ bv.bv_len = snprintf( digbuf, sizeof(digbuf), "%u,%u", ++ ai->ai_multi_hi, ai->ai_multi_lo ); ++ if ( bv.bv_len ) { ++ bv.bv_len += ai->ai_desc->ad_cname.bv_len + 1; ++ ptr = ch_malloc( bv.bv_len+1 ); ++ bv.bv_val = lutil_strcopy( ptr, ai->ai_desc->ad_cname.bv_val ); ++ *bv.bv_val++ = ' '; ++ strcpy(bv.bv_val, digbuf); ++ bv.bv_val = ptr; ++ ber_bvarray_add( bva, &bv ); ++ } ++ return 0; ++} ++ ++void ++mdb_attr_multi_unparse( struct mdb_info *mdb, BerVarray *bva ) ++{ ++ int i; ++ ++ if ( mdb->mi_multi_hi < UINT_MAX ) { ++ aidef.ai_multi_hi = mdb->mi_multi_hi; ++ aidef.ai_multi_lo = mdb->mi_multi_lo; ++ mdb_attr_multi_unparser( &aidef, bva ); ++ } ++ for ( i=0; imi_nattrs; i++ ) ++ if ( mdb->mi_attrs[i]->ai_multi_hi < UINT_MAX ) ++ mdb_attr_multi_unparser( mdb->mi_attrs[i], bva ); ++} ++ ++void ++mdb_attr_multi_thresh( struct mdb_info *mdb, AttributeDescription *ad, unsigned *hi, unsigned *lo ) ++{ ++ AttrInfo *ai = mdb_attr_mask( mdb, ad ); ++ if ( ai && ai->ai_multi_hi < UINT_MAX ) ++ { ++ if ( hi ) ++ *hi = ai->ai_multi_hi; ++ if ( lo ) ++ *lo = ai->ai_multi_lo; ++ } else ++ { ++ if ( hi ) ++ *hi = mdb->mi_multi_hi; ++ if ( lo ) ++ *lo = mdb->mi_multi_lo; ++ } + } + + void +@@ -532,12 +706,18 @@ + + for ( i=0; imi_nattrs; i++ ) { + if ( mdb->mi_attrs[i]->ai_indexmask & MDB_INDEX_DELETING ) { +- int j; +- mdb_attr_info_free( mdb->mi_attrs[i] ); +- mdb->mi_nattrs--; +- for (j=i; jmi_nattrs; j++) +- mdb->mi_attrs[j] = mdb->mi_attrs[j+1]; +- i--; ++ /* if this is also a multival rec, just clear index */ ++ if ( mdb->mi_attrs[i]->ai_multi_lo < UINT_MAX ) { ++ mdb->mi_attrs[i]->ai_indexmask = 0; ++ mdb->mi_attrs[i]->ai_newmask = 0; ++ } else { ++ int j; ++ mdb_attr_info_free( mdb->mi_attrs[i] ); ++ mdb->mi_nattrs--; ++ for (j=i; jmi_nattrs; j++) ++ mdb->mi_attrs[j] = mdb->mi_attrs[j+1]; ++ i--; ++ } + } + } + } +--- openldap-2.4.46/servers/slapd/back-mdb/back-mdb.h.orig 2018-08-31 21:34:38.687225761 +0000 ++++ openldap-2.4.46/servers/slapd/back-mdb/back-mdb.h 2018-08-31 21:20:15.249014871 +0000 +@@ -32,7 +32,8 @@ #define MDB_AD2ID 0 #define MDB_DN2ID 1 #define MDB_ID2ENTRY 2 @@ -51,7 +285,7 @@ index b271069..f453a74 100644 /* The default search IDL stack cache depth */ #define DEFAULT_SEARCH_STACK_DEPTH 16 -@@ -85,6 +86,7 @@ struct mdb_info { +@@ -85,6 +86,7 @@ int mi_txn_cp; uint32_t mi_txn_cp_min; uint32_t mi_txn_cp_kbyte; @@ -59,7 +293,7 @@ index b271069..f453a74 100644 struct re_s *mi_txn_cp_task; struct re_s *mi_index_task; -@@ -104,6 +106,13 @@ struct mdb_info { +@@ -104,6 +106,13 @@ int mi_numads; @@ -73,7 +307,7 @@ index b271069..f453a74 100644 MDB_dbi mi_dbis[MDB_NDB]; AttributeDescription *mi_ads[MDB_MAXADS]; int mi_adxs[MDB_MAXADS]; -@@ -112,6 +121,7 @@ struct mdb_info { +@@ -112,6 +121,7 @@ #define mi_id2entry mi_dbis[MDB_ID2ENTRY] #define mi_dn2id mi_dbis[MDB_DN2ID] #define mi_ad2id mi_dbis[MDB_AD2ID] @@ -81,7 +315,7 @@ index b271069..f453a74 100644 typedef struct mdb_op_info { OpExtra moi_oe; -@@ -122,24 +132,6 @@ typedef struct mdb_op_info { +@@ -122,24 +132,6 @@ #define MOI_READER 0x01 #define MOI_FREEIT 0x02 @@ -106,41 +340,138 @@ index b271069..f453a74 100644 LDAP_END_DECL /* for the cache of attribute information (which are indexed, etc.) */ -diff --git a/servers/slapd/back-mdb/config.c b/servers/slapd/back-mdb/config.c -index d4c6f96..bb9c135 100644 ---- a/servers/slapd/back-mdb/config.c -+++ b/servers/slapd/back-mdb/config.c -@@ -78,6 +78,16 @@ static ConfigTable mdbcfg[] = { +@@ -156,6 +148,8 @@ + MDB_cursor *ai_cursor; /* for tools */ + int ai_idx; /* position in AI array */ + MDB_dbi ai_dbi; ++ unsigned ai_multi_hi; ++ unsigned ai_multi_lo; + } AttrInfo; + + /* These flags must not clash with SLAP_INDEX flags or ops in slap.h! */ +--- openldap-2.4.46/servers/slapd/back-mdb/config.c.orig 2018-08-31 21:34:31.111309728 +0000 ++++ openldap-2.4.46/servers/slapd/back-mdb/config.c 2018-08-31 21:22:48.499227430 +0000 +@@ -39,7 +39,8 @@ + MDB_MAXREADERS, + MDB_MAXSIZE, + MDB_MODE, +- MDB_SSTACK ++ MDB_SSTACK, ++ MDB_MULTIVAL, + }; + + static ConfigTable mdbcfg[] = { +@@ -78,6 +79,11 @@ mdb_cf_gen, "( OLcfgDbAt:0.3 NAME 'olcDbMode' " "DESC 'Unix permissions of database files' " "SYNTAX OMsDirectoryString SINGLE-VALUE )", NULL, NULL }, -+ { "multival_hi", "num", 2, 2, 0, ARG_UINT|ARG_OFFSET, -+ (void *)offsetof(struct mdb_info, mi_multi_hi), -+ "( OLcfgDbAt:12.6 NAME 'olcDbMultivalHi' " -+ "DESC 'Threshold for splitting multivalued attr out of main blob' " -+ "SYNTAX OMsInteger SINGLE-VALUE )", NULL, NULL }, -+ { "multival_lo", "num", 2, 2, 0, ARG_UINT|ARG_OFFSET, -+ (void *)offsetof(struct mdb_info, mi_multi_lo), -+ "( OLcfgDbAt:12.7 NAME 'olcDbMultivalLo' " -+ "DESC 'Threshold for consolidating multivalued attr back into main blob' " -+ "SYNTAX OMsInteger SINGLE-VALUE )", NULL, NULL }, ++ { "multival", "attr> value_ulong = mdb->mi_mapsize; + break; ++ ++ case MDB_MULTIVAL: ++ mdb_attr_multi_unparse( mdb, &c->rvalue_vals ); ++ if ( !c->rvalue_vals ) rc = 1; ++ break; + } + return rc; + } else if ( c->op == LDAP_MOD_DELETE ) { +@@ -473,6 +486,61 @@ + } + } + break; ++ case MDB_MULTIVAL: ++ if ( c->valx == -1 ) { ++ int i; ++ ++ /* delete all */ ++ for ( i = 0; i < mdb->mi_nattrs; i++ ) { ++ mdb->mi_attrs[i]->ai_multi_hi = UINT_MAX; ++ mdb->mi_attrs[i]->ai_multi_lo = UINT_MAX; ++ } ++ mdb->mi_multi_hi = UINT_MAX; ++ mdb->mi_multi_lo = UINT_MAX; ++ ++ } else { ++ struct berval bv, def = BER_BVC("default"); ++ char *ptr; ++ ++ for (ptr = c->line; !isspace( (unsigned char) *ptr ); ptr++); ++ ++ bv.bv_val = c->line; ++ bv.bv_len = ptr - bv.bv_val; ++ if ( bvmatch( &bv, &def )) { ++ mdb->mi_multi_hi = UINT_MAX; ++ mdb->mi_multi_lo = UINT_MAX; ++ ++ } else { ++ int i; ++ char **attrs; ++ char sep; ++ ++ sep = bv.bv_val[ bv.bv_len ]; ++ bv.bv_val[ bv.bv_len ] = '\0'; ++ attrs = ldap_str2charray( bv.bv_val, "," ); ++ ++ for ( i = 0; attrs[ i ]; i++ ) { ++ AttributeDescription *ad = NULL; ++ const char *text; ++ AttrInfo *ai; ++ ++ slap_str2ad( attrs[ i ], &ad, &text ); ++ /* if we got here... */ ++ assert( ad != NULL ); ++ ++ ai = mdb_attr_mask( mdb, ad ); ++ /* if we got here... */ ++ assert( ai != NULL ); ++ ++ ai->ai_multi_hi = UINT_MAX; ++ ai->ai_multi_lo = UINT_MAX; ++ } ++ ++ bv.bv_val[ bv.bv_len ] = sep; ++ ldap_charray_free( attrs ); ++ } ++ } ++ break; + } + return rc; + } +@@ -678,6 +746,12 @@ + } + break; + ++ case MDB_MULTIVAL: ++ rc = mdb_attr_multi_config( mdb, c->fname, c->lineno, ++ c->argc - 1, &c->argv[1], &c->reply); ++ ++ if( rc != LDAP_SUCCESS ) return 1; ++ break; + } + return 0; + } +--- openldap-2.4.46/servers/slapd/back-mdb/id2entry.c.orig 2018-08-31 21:34:57.455017822 +0000 ++++ openldap-2.4.46/servers/slapd/back-mdb/id2entry.c 2018-08-31 21:20:15.253014824 +0000 @@ -23,10 +23,12 @@ #include "back-mdb.h" @@ -155,7 +486,7 @@ index b580ebd..ea2129a 100644 } Ecount; static int mdb_entry_partsize(struct mdb_info *mdb, MDB_txn *txn, Entry *e, -@@ -35,6 +37,229 @@ static int mdb_entry_encode(Operation *op, Entry *e, MDB_val *data, +@@ -35,6 +37,229 @@ Ecount *ec); static Entry *mdb_entry_alloc( Operation *op, int nattrs, int nvals ); @@ -385,7 +716,7 @@ index b580ebd..ea2129a 100644 #define ADD_FLAGS (MDB_NOOVERWRITE|MDB_APPEND) static int mdb_id2entry_put( -@@ -47,7 +272,7 @@ static int mdb_id2entry_put( +@@ -47,7 +272,7 @@ struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private; Ecount ec; MDB_val key, data; @@ -394,7 +725,7 @@ index b580ebd..ea2129a 100644 /* We only store rdns, and they go in the dn2id database. */ -@@ -64,7 +289,7 @@ static int mdb_id2entry_put( +@@ -64,7 +289,7 @@ flag &= ~MDB_APPEND; again: @@ -403,7 +734,7 @@ index b580ebd..ea2129a 100644 if ( mc ) rc = mdb_cursor_put( mc, &key, &data, flag ); else -@@ -73,6 +298,31 @@ again: +@@ -73,6 +298,31 @@ rc = mdb_entry_encode( op, e, &data, &ec ); if( rc != LDAP_SUCCESS ) return rc; @@ -493,7 +824,7 @@ index b580ebd..ea2129a 100644 return rc; } -@@ -528,11 +798,13 @@ +@@ -528,11 +798,14 @@ static int mdb_entry_partsize(struct mdb_info *mdb, MDB_txn *txn, Entry *e, Ecount *eh) { @@ -502,6 +833,7 @@ index b580ebd..ea2129a 100644 + ber_len_t len, dlen; + int i, nat = 0, nval = 0, nnval = 0, doff = 0; Attribute *a; ++ unsigned hi; + eh->multi = NULL; len = 4*sizeof(int); /* nattrs, nvals, ocflags, offset */ @@ -509,13 +841,14 @@ index b580ebd..ea2129a 100644 for (a=e->e_attrs; a; a=a->a_next) { /* For AttributeDesc, we only store the attr index */ nat++; -@@ -547,46 +819,74 @@ +@@ -547,46 +820,75 @@ return rc; } len += 2*sizeof(int); /* AD index, numvals */ + dlen += 2*sizeof(int); nval += a->a_numvals + 1; /* empty berval at end */ -+ if (a->a_numvals > mdb->mi_multi_hi) ++ mdb_attr_multi_thresh( mdb, a->a_desc, &hi, NULL ); ++ if (a->a_numvals > hi) + a->a_flags |= SLAP_ATTR_BIG_MULTI; + if (a->a_flags & SLAP_ATTR_BIG_MULTI) + doff += a->a_numvals; @@ -601,7 +934,7 @@ index b580ebd..ea2129a 100644 */ static int mdb_entry_encode(Operation *op, Entry *e, MDB_val *data, Ecount *eh) { -@@ -614,31 +914,37 @@ +@@ -614,31 +916,37 @@ if (!a->a_desc->ad_index) return LDAP_UNDEFINED_TYPE; l = mdb->mi_adxs[a->a_desc->ad_index]; @@ -656,7 +989,7 @@ index b580ebd..ea2129a 100644 } } } -@@ -656,7 +962,7 @@ +@@ -656,7 +964,7 @@ * structure. Attempting to do so will likely corrupt memory. */ @@ -665,7 +998,7 @@ index b580ebd..ea2129a 100644 { struct mdb_info *mdb = (struct mdb_info *) op->o_bd->be_private; int i, j, nattrs, nvals; -@@ -667,6 +973,7 @@ +@@ -667,6 +975,7 @@ unsigned int *lp = (unsigned int *)data->mv_data; unsigned char *ptr; BerVarray bptr; @@ -673,7 +1006,7 @@ index b580ebd..ea2129a 100644 Debug( LDAP_DEBUG_TRACE, "=> mdb_entry_decode:\n", -@@ -685,43 +992,49 @@ +@@ -685,43 +994,49 @@ ptr = (unsigned char *)(lp + i); for (;nattrs>0; nattrs--) { @@ -742,7 +1075,7 @@ index b580ebd..ea2129a 100644 for (i=0; ia_numvals; i++) { bptr->bv_len = *lp++; bptr->bv_val = (char *)ptr; -@@ -731,9 +1044,23 @@ +@@ -731,9 +1046,23 @@ bptr->bv_val = NULL; bptr->bv_len = 0; bptr++; @@ -768,7 +1101,7 @@ index b580ebd..ea2129a 100644 /* FIXME: This is redundant once a sorted entry is saved into the DB */ if (( a->a_desc->ad_type->sat_flags & SLAP_AT_SORTED_VAL ) && !(a->a_flags & SLAP_ATTR_SORTED_VALS)) { -@@ -745,7 +1072,7 @@ +@@ -745,7 +1074,7 @@ Debug( LDAP_DEBUG_ANY, "mdb_entry_decode: attributeType %s value #%d provided more than once\n", a->a_desc->ad_cname.bv_val, j, 0 ); @@ -777,7 +1110,7 @@ index b580ebd..ea2129a 100644 } } a->a_next = a+1; -@@ -753,9 +1080,13 @@ +@@ -753,9 +1082,13 @@ } a[-1].a_next = NULL; done: @@ -793,11 +1126,29 @@ index b580ebd..ea2129a 100644 + mdb_cursor_close(mvc); + return rc; } -diff --git a/servers/slapd/back-mdb/init.c b/servers/slapd/back-mdb/init.c -index 09e097c..a197e81 100644 ---- a/servers/slapd/back-mdb/init.c -+++ b/servers/slapd/back-mdb/init.c -@@ -31,6 +31,7 @@ static const struct berval mdmi_databases[] = { +--- openldap-2.4.46/servers/slapd/back-mdb/index.c.orig 2018-08-31 21:34:11.079531839 +0000 ++++ openldap-2.4.46/servers/slapd/back-mdb/index.c 2018-08-31 21:20:15.253014824 +0000 +@@ -310,7 +310,7 @@ + /* If this type has no AD, we've never used it before */ + if( type->sat_ad ) { + ai = mdb_attr_mask( op->o_bd->be_private, type->sat_ad ); +- if ( ai ) { ++ if ( ai && ( ai->ai_indexmask || ai->ai_newmask )) { + #ifdef LDAP_COMP_MATCH + /* component indexing */ + if ( ai->ai_cr ) { +@@ -349,7 +349,7 @@ + if( desc ) { + ai = mdb_attr_mask( op->o_bd->be_private, desc ); + +- if( ai ) { ++ if( ai && ( ai->ai_indexmask || ai->ai_newmask )) { + if ( opid == MDB_INDEX_UPDATE_OP ) + mask = ai->ai_newmask & ~ai->ai_indexmask; + else +--- openldap-2.4.46/servers/slapd/back-mdb/init.c.orig 2018-08-31 21:34:16.043476787 +0000 ++++ openldap-2.4.46/servers/slapd/back-mdb/init.c 2018-08-31 21:20:08.585093409 +0000 +@@ -31,6 +31,7 @@ BER_BVC("ad2i"), BER_BVC("dn2i"), BER_BVC("id2e"), @@ -805,7 +1156,7 @@ index 09e097c..a197e81 100644 BER_BVNULL }; -@@ -63,6 +64,8 @@ mdb_db_init( BackendDB *be, ConfigReply *cr ) +@@ -63,6 +64,8 @@ mdb->mi_mapsize = DEFAULT_MAPSIZE; mdb->mi_rtxn_size = DEFAULT_RTXN_SIZE; @@ -814,7 +1165,7 @@ index 09e097c..a197e81 100644 be->be_private = mdb; be->be_cf_ocs = be->bd_info->bi_cf_ocs; -@@ -201,6 +204,8 @@ mdb_db_open( BackendDB *be, ConfigReply *cr ) +@@ -201,6 +204,8 @@ } else { if ( i == MDB_DN2ID ) flags |= MDB_DUPSORT; @@ -823,7 +1174,7 @@ index 09e097c..a197e81 100644 if ( !(slapMode & SLAP_TOOL_READONLY) ) flags |= MDB_CREATE; } -@@ -224,7 +229,10 @@ mdb_db_open( BackendDB *be, ConfigReply *cr ) +@@ -224,7 +229,10 @@ if ( i == MDB_ID2ENTRY ) mdb_set_compare( txn, mdb->mi_dbis[i], mdb_id_compare ); @@ -835,11 +1186,9 @@ index 09e097c..a197e81 100644 MDB_cursor *mc; MDB_val key, data; mdb_set_dupsort( txn, mdb->mi_dbis[i], mdb_dup_compare ); -diff --git a/servers/slapd/back-mdb/modify.c b/servers/slapd/back-mdb/modify.c -index 4509373..c860cf3 100644 ---- a/servers/slapd/back-mdb/modify.c -+++ b/servers/slapd/back-mdb/modify.c -@@ -74,13 +74,17 @@ int mdb_modify_internal( +--- openldap-2.4.46/servers/slapd/back-mdb/modify.c.orig 2018-08-31 21:35:11.854858351 +0000 ++++ openldap-2.4.46/servers/slapd/back-mdb/modify.c 2018-08-31 21:20:15.253014824 +0000 +@@ -74,13 +74,17 @@ char *textbuf, size_t textlen ) { @@ -858,7 +1207,7 @@ index 4509373..c860cf3 100644 Debug( LDAP_DEBUG_TRACE, "mdb_modify_internal: 0x%08lx: %s\n", e->e_id, e->e_dn, 0); -@@ -129,16 +133,69 @@ int mdb_modify_internal( +@@ -129,16 +133,71 @@ mod = &ml->sml_mod; got_delete = 0; @@ -893,12 +1242,14 @@ index 4509373..c860cf3 100644 Debug(LDAP_DEBUG_ARGS, "mdb_modify_internal: %d %s\n", err, *text, 0); + } else { ++ unsigned hi; + if (!aold) + anew = attr_find( e->e_attrs, mod->sm_desc ); + else + anew = aold; ++ mdb_attr_multi_thresh( mdb, mod->sm_desc, &hi, NULL ); + /* check for big multivalued attrs */ -+ if ( anew->a_numvals > mdb->mi_multi_hi ) ++ if ( anew->a_numvals > hi ) + anew->a_flags |= SLAP_ATTR_BIG_MULTI; + if ( anew->a_flags & SLAP_ATTR_BIG_MULTI ) { + if (!mvc) { @@ -928,7 +1279,7 @@ index 4509373..c860cf3 100644 } break; -@@ -148,16 +205,59 @@ int mdb_modify_internal( +@@ -148,16 +207,61 @@ break; } @@ -966,7 +1317,9 @@ index 4509373..c860cf3 100644 + if ( mod->sm_numvals ) { + anew = attr_find( e->e_attrs, mod->sm_desc ); + if ( anew ) { -+ if ( anew->a_numvals < mdb->mi_multi_lo ) { ++ unsigned lo; ++ mdb_attr_multi_thresh( mdb, mod->sm_desc, NULL, &lo ); ++ if ( anew->a_numvals < lo ) { + anew->a_flags ^= SLAP_ATTR_BIG_MULTI; + anew = NULL; + } else { @@ -989,9 +1342,11 @@ index 4509373..c860cf3 100644 } break; -@@ -172,6 +272,36 @@ int mdb_modify_internal( +@@ -171,7 +275,39 @@ + Debug(LDAP_DEBUG_ARGS, "mdb_modify_internal: %d %s\n", err, *text, 0); } else { ++ unsigned hi; got_delete = 1; + if (a_flags & SLAP_ATTR_BIG_MULTI) { + Attribute a_dummy; @@ -1009,7 +1364,8 @@ index 4509373..c860cf3 100644 + goto mval_fail; + } + anew = attr_find( e->e_attrs, mod->sm_desc ); -+ if (mod->sm_numvals > mdb->mi_multi_hi) { ++ mdb_attr_multi_thresh( mdb, mod->sm_desc, &hi, NULL ); ++ if (mod->sm_numvals > hi) { + anew->a_flags |= SLAP_ATTR_BIG_MULTI; + if (!mvc) { + err = mdb_cursor_open( tid, mdb->mi_dbis[MDB_ID2VAL], &mvc ); @@ -1026,7 +1382,7 @@ index 4509373..c860cf3 100644 } break; -@@ -198,21 +328,9 @@ int mdb_modify_internal( +@@ -198,21 +334,9 @@ * We need to add index if necessary. */ mod->sm_op = LDAP_MOD_ADD; @@ -1051,7 +1407,7 @@ index 4509373..c860cf3 100644 case SLAP_MOD_SOFTDEL: Debug(LDAP_DEBUG_ARGS, -@@ -222,23 +340,8 @@ int mdb_modify_internal( +@@ -222,23 +346,8 @@ * We need to add index if necessary. */ mod->sm_op = LDAP_MOD_DELETE; @@ -1077,7 +1433,7 @@ index 4509373..c860cf3 100644 case SLAP_MOD_ADD_IF_NOT_PRESENT: if ( attr_find( e->e_attrs, mod->sm_desc ) != NULL ) { -@@ -254,17 +357,9 @@ int mdb_modify_internal( +@@ -254,17 +363,9 @@ * We need to add index if necessary. */ mod->sm_op = LDAP_MOD_ADD; @@ -1098,11 +1454,25 @@ index 4509373..c860cf3 100644 default: Debug(LDAP_DEBUG_ANY, "mdb_modify_internal: invalid op %d\n", -diff --git a/servers/slapd/back-mdb/proto-mdb.h b/servers/slapd/back-mdb/proto-mdb.h -index 5508996..e12439f 100644 ---- a/servers/slapd/back-mdb/proto-mdb.h -+++ b/servers/slapd/back-mdb/proto-mdb.h -@@ -168,6 +168,9 @@ int mdb_filter_candidates( +--- openldap-2.4.46/servers/slapd/back-mdb/proto-mdb.h.orig 2018-08-31 21:34:46.931134408 +0000 ++++ openldap-2.4.46/servers/slapd/back-mdb/proto-mdb.h 2018-08-31 21:20:15.253014824 +0000 +@@ -44,6 +44,15 @@ + void mdb_attr_index_free LDAP_P(( struct mdb_info *mdb, + AttributeDescription *ad )); + ++int mdb_attr_multi_config LDAP_P(( struct mdb_info *mdb, ++ const char *fname, int lineno, ++ int argc, char **argv, struct config_reply_s *cr )); ++ ++void mdb_attr_multi_unparse LDAP_P(( struct mdb_info *mdb, BerVarray *bva )); ++ ++void mdb_attr_multi_thresh LDAP_P(( struct mdb_info *mdb, AttributeDescription *ad, ++ unsigned *hi, unsigned *lo )); ++ + void mdb_attr_info_free( AttrInfo *ai ); + + int mdb_ad_read( struct mdb_info *mdb, MDB_txn *txn ); +@@ -168,6 +177,9 @@ * id2entry.c */ @@ -1112,7 +1482,7 @@ index 5508996..e12439f 100644 int mdb_id2entry_add( Operation *op, MDB_txn *tid, -@@ -201,11 +204,14 @@ int mdb_entry_return( Operation *op, Entry *e ); +@@ -201,11 +213,14 @@ BI_entry_release_rw mdb_entry_release; BI_entry_get_rw mdb_entry_get; @@ -1128,11 +1498,9 @@ index 5508996..e12439f 100644 /* * idl.c */ -diff --git a/servers/slapd/back-mdb/search.c b/servers/slapd/back-mdb/search.c -index a4d58e4..dc7b33a 100644 ---- a/servers/slapd/back-mdb/search.c -+++ b/servers/slapd/back-mdb/search.c -@@ -708,6 +708,7 @@ dn2entry_retry: +--- openldap-2.4.46/servers/slapd/back-mdb/search.c.orig 2018-08-31 21:34:04.503604782 +0000 ++++ openldap-2.4.46/servers/slapd/back-mdb/search.c 2018-08-31 21:20:08.589093362 +0000 +@@ -708,6 +708,7 @@ } wwctx.flag = 0; @@ -1140,7 +1508,7 @@ index a4d58e4..dc7b33a 100644 /* If we're running in our own read txn */ if ( moi == &opinfo ) { cb.sc_writewait = mdb_writewait; -@@ -921,7 +922,7 @@ notfound: +@@ -921,7 +922,7 @@ goto done; } @@ -1149,11 +1517,9 @@ index a4d58e4..dc7b33a 100644 if ( rs->sr_err ) { rs->sr_err = LDAP_OTHER; rs->sr_text = "internal error in mdb_entry_decode"; -diff --git a/servers/slapd/back-mdb/tools.c b/servers/slapd/back-mdb/tools.c -index bb56e65..559c3f8 100644 ---- a/servers/slapd/back-mdb/tools.c -+++ b/servers/slapd/back-mdb/tools.c -@@ -379,7 +379,7 @@ mdb_tool_entry_get_int( BackendDB *be, ID id, Entry **ep ) +--- openldap-2.4.46/servers/slapd/back-mdb/tools.c.orig 2018-08-31 21:34:26.027366088 +0000 ++++ openldap-2.4.46/servers/slapd/back-mdb/tools.c 2018-08-31 21:20:08.589093362 +0000 +@@ -379,7 +379,7 @@ } } } @@ -1162,20 +1528,3 @@ index bb56e65..559c3f8 100644 e->e_id = id; if ( !BER_BVISNULL( &dn )) { e->e_name = dn; -diff --git a/servers/slapd/slap.h b/servers/slapd/slap.h -index dc186fe..d6c0ba6 100644 ---- a/servers/slapd/slap.h -+++ b/servers/slapd/slap.h -@@ -1163,10 +1163,11 @@ struct Attribute { - #define SLAP_ATTR_DONT_FREE_DATA 0x4U - #define SLAP_ATTR_DONT_FREE_VALS 0x8U - #define SLAP_ATTR_SORTED_VALS 0x10U /* values are sorted */ -+#define SLAP_ATTR_BIG_MULTI 0x20U /* for backends */ - - /* These flags persist across an attr_dup() */ - #define SLAP_ATTR_PERSISTENT_FLAGS \ -- SLAP_ATTR_SORTED_VALS -+ (SLAP_ATTR_SORTED_VALS|SLAP_ATTR_BIG_MULTI) - - Attribute *a_next; - #ifdef LDAP_COMP_MATCH diff --git a/thirdparty/openldap/zimbra-openldap/debian/changelog b/thirdparty/openldap/zimbra-openldap/debian/changelog index b7f10ed1c..c7b6e3d39 100644 --- a/thirdparty/openldap/zimbra-openldap/debian/changelog +++ b/thirdparty/openldap/zimbra-openldap/debian/changelog @@ -1,3 +1,10 @@ +zimbra-openldap (VERSION-1zimbra8.7b3ZAPPEND) unstable; urgency=medium + + * Update multival for new syntax that allows the capability to work around LDMB 0.9 limitations + * Update liblmdb library to have a larger max keysize + + -- Zimbra Packaging Services Fri, 31 Aug 2018 14:52:31 +0000 + zimbra-openldap (VERSION-1zimbra8.7b2ZAPPEND) unstable; urgency=medium * Update multival patch to make sure a->a_numvals matches id2v counts diff --git a/thirdparty/openldap/zimbra-openldap/debian/patches/series b/thirdparty/openldap/zimbra-openldap/debian/patches/series index c6b4dc4d5..b80d6084d 100644 --- a/thirdparty/openldap/zimbra-openldap/debian/patches/series +++ b/thirdparty/openldap/zimbra-openldap/debian/patches/series @@ -5,3 +5,5 @@ ITS8054.patch ITS8843.patch liblmdb-soname.patch multival.patch +liblmdb-keysize.patch +index-delete.patch diff --git a/thirdparty/openldap/zimbra-openldap/rpm/SPECS/openldap.spec b/thirdparty/openldap/zimbra-openldap/rpm/SPECS/openldap.spec index 4a437b25c..9bcfce464 100644 --- a/thirdparty/openldap/zimbra-openldap/rpm/SPECS/openldap.spec +++ b/thirdparty/openldap/zimbra-openldap/rpm/SPECS/openldap.spec @@ -1,7 +1,7 @@ Summary: Zimbra's openldap build Name: zimbra-openldap Version: VERSION -Release: 1zimbra8.7b2ZAPPEND +Release: 1zimbra8.7b3ZAPPEND License: BSD Source: %{name}-%{version}.tgz Patch0: ITS5037.patch @@ -11,6 +11,8 @@ Patch3: ITS8054.patch Patch4: ITS8843.patch Patch5: liblmdb-soname.patch Patch6: multival.patch +Patch7: liblmdb-keysize.patch +Patch8: index-delete.patch BuildRequires: zimbra-openssl-devel BuildRequires: zimbra-cyrus-sasl-devel BuildRequires: zimbra-libltdl-devel @@ -21,6 +23,9 @@ URL: http://www.openldap.org The Zimbra openldap build %changelog +* Fri Aug 31 2018 Zimbra Packaging Services - VERSION-1zimbra8.7b3ZAPPEND +- Update multival for new syntax that allows the capability to work around LDMB 0.9 limitations +- Update liblmdb library to have a larger max keysize * Wed Jul 18 2018 Zimbra Packaging Services - VERSION-1zimbra8.7b2ZAPPEND - Update multival patch to make sure a->a_numvals matches id2v counts * Tue Apr 24 2018 Zimbra Packaging Services - VERSION-1zimbra8.7b1ZAPPEND @@ -37,6 +42,8 @@ The Zimbra openldap build %patch4 -p1 %patch5 -p1 %patch6 -p1 +%patch7 -p1 +%patch8 -p1 %build # Alternate Makeargs: DEFINES="-DCHECK_CSN -DSLAP_SCHEMA_EXPOSE -DMDB_DEBUG=3" From a72cb370e0cfe32f5b8535d194d64ca6845ce85c Mon Sep 17 00:00:00 2001 From: Amit Srivastava Date: Tue, 18 Sep 2018 16:45:05 +0530 Subject: [PATCH 10/15] ZCS-5976: Bumped up ldap-core-components for openldap 2.4.46 --- .../zimbra-ldap-components/debian/changelog | 8 +++++++- .../zimbra-ldap-components/rpm/SPECS/ldap-components.spec | 4 +++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/zimbra/ldap-components/zimbra-ldap-components/debian/changelog b/zimbra/ldap-components/zimbra-ldap-components/debian/changelog index 68b47c3e6..50e5518b6 100644 --- a/zimbra/ldap-components/zimbra-ldap-components/debian/changelog +++ b/zimbra/ldap-components/zimbra-ldap-components/debian/changelog @@ -1,8 +1,14 @@ +zimbra-ldap-components (1.0.2-ITERATIONZAPPEND) unstable; urgency=low + + * Updated openldap 2.4.47 + + -- Zimbra Packaging Services Wed, 18 Sep 2018 00:00:00 +0000 + zimbra-ldap-components (1.0.1-ITERATIONZAPPEND) unstable; urgency=low * Updated openldap 2.4.46 - -- Zimbra Packaging Services Wed, 25 Sep 2018 00:00:00 +0000 + -- Zimbra Packaging Services Wed, 25 Jul 2018 00:00:00 +0000 zimbra-ldap-components (1.0.0-ITERATIONZAPPEND) unstable; urgency=low diff --git a/zimbra/ldap-components/zimbra-ldap-components/rpm/SPECS/ldap-components.spec b/zimbra/ldap-components/zimbra-ldap-components/rpm/SPECS/ldap-components.spec index d7d0d2b7b..7305c1b4e 100644 --- a/zimbra/ldap-components/zimbra-ldap-components/rpm/SPECS/ldap-components.spec +++ b/zimbra/ldap-components/zimbra-ldap-components/rpm/SPECS/ldap-components.spec @@ -1,6 +1,6 @@ Summary: Zimbra components for ldap package Name: zimbra-ldap-components -Version: 1.0.1 +Version: 1.0.2 Release: ITERATIONZAPPEND License: GPL-2 Requires: zimbra-ldap-base, zimbra-lmdb >= 2.4.46-1zimbra8.7b2ZAPPEND @@ -16,6 +16,8 @@ Zimbra ldap components pulls in all the packages used by zimbra-ldap %changelog +* Tue Sep 18 2018 Zimbra Packaging Services - 1.0.2 +- Updated Updated openldap 2.4.47 * Wed Jul 25 2018 Zimbra Packaging Services - 1.0.1 - Updated Updated openldap 2.4.46 * Wed Sep 09 2015 Zimbra Packaging Services - 1.0.0 From 7e08d7a5803847f177ce614d8f9ffec02b65b5b0 Mon Sep 17 00:00:00 2001 From: Amit Srivastava Date: Wed, 19 Sep 2018 18:20:33 +0530 Subject: [PATCH 11/15] ZCS-5976: Bumped up ldap-core-components for openldap 2.4.46 for zimbra8.7b3 --- zimbra/ldap-components/zimbra-ldap-components/debian/control | 2 +- .../zimbra-ldap-components/rpm/SPECS/ldap-components.spec | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/zimbra/ldap-components/zimbra-ldap-components/debian/control b/zimbra/ldap-components/zimbra-ldap-components/debian/control index e816edeb2..893829944 100644 --- a/zimbra/ldap-components/zimbra-ldap-components/debian/control +++ b/zimbra/ldap-components/zimbra-ldap-components/debian/control @@ -8,7 +8,7 @@ Standards-Version: 3.9.5 Package: zimbra-ldap-components Architecture: all Depends: ${misc:Depends}, ${perl:Depends}, ${shlibs:Depends}, - zimbra-ldap-base, zimbra-lmdb (>= 2.4.46-1zimbra8.7b2ZAPPEND), zimbra-openldap-server (>= 2.4.46-1zimbra8.7b2ZAPPEND) + zimbra-ldap-base, zimbra-lmdb (>= 2.4.46-1zimbra8.7b3ZAPPEND), zimbra-openldap-server (>= 2.4.46-1zimbra8.7b3ZAPPEND) Description: Zimbra components for ldap package Zimbra ldap components pulls in all the packages used by zimbra-ldap diff --git a/zimbra/ldap-components/zimbra-ldap-components/rpm/SPECS/ldap-components.spec b/zimbra/ldap-components/zimbra-ldap-components/rpm/SPECS/ldap-components.spec index 7305c1b4e..a057ab388 100644 --- a/zimbra/ldap-components/zimbra-ldap-components/rpm/SPECS/ldap-components.spec +++ b/zimbra/ldap-components/zimbra-ldap-components/rpm/SPECS/ldap-components.spec @@ -3,8 +3,8 @@ Name: zimbra-ldap-components Version: 1.0.2 Release: ITERATIONZAPPEND License: GPL-2 -Requires: zimbra-ldap-base, zimbra-lmdb >= 2.4.46-1zimbra8.7b2ZAPPEND -Requires: zimbra-openldap-server >= 2.4.46-1zimbra8.7b2ZAPPEND +Requires: zimbra-ldap-base, zimbra-lmdb >= 2.4.46-1zimbra8.7b3ZAPPEND +Requires: zimbra-openldap-server >= 2.4.46-1zimbra8.7b3ZAPPEND Packager: Zimbra Packaging Services Group: Development/Languages AutoReqProv: no From 77a742c9b9f6fe8f648e86f4a347bff9f75ff874 Mon Sep 17 00:00:00 2001 From: Amit Srivastava Date: Thu, 18 Oct 2018 19:20:39 +0530 Subject: [PATCH 12/15] ZCS-6173: To build aspell for ubuntu 18 have to updated the version. https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=853320 Bug refference http://aspell.net/aspell-0.60.7-rc1.txt detail having the bug fix information. --- thirdparty/aspell/Makefile | 2 +- versions.def | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/thirdparty/aspell/Makefile b/thirdparty/aspell/Makefile index 9170ee6d1..43f7154d2 100644 --- a/thirdparty/aspell/Makefile +++ b/thirdparty/aspell/Makefile @@ -6,7 +6,7 @@ pvers := $(ASPELL_VERSION) pname := aspell pfile := $(pname)-$(pvers).tar.gz psrc_file := $(SRC_DIR)/$(pfile) -purl := ftp://ftp.gnu.org/gnu/aspell/$(pfile) +purl := ftp://alpha.gnu.org/gnu/aspell/$(pfile) zname := zimbra-$(pname) zspec := $(pname).spec diff --git a/versions.def b/versions.def index 8a14466eb..6e9ea5711 100644 --- a/versions.def +++ b/versions.def @@ -6,7 +6,7 @@ APR_VERSION := 1.5.2 APR_UTIL_VERSION := 1.5.4 -ASPELL_VERSION := 0.60.6.1 +ASPELL_VERSION := 0.60.7-rc1 ASPELL-AR_VERSION := 1.2-0 ASPELL-AR_NORM := 1.2.0 ASPELL-DA_VERSION := 1.4.42-1 From 876f4393f594179b1d40b6c58d9b0057e3c8e882 Mon Sep 17 00:00:00 2001 From: Prashant Surana Date: Mon, 12 Nov 2018 09:27:25 +0000 Subject: [PATCH 13/15] ZCS-6172: Building zimbra-openjdk from openjdk 11 tag 28 All the changes made was for the Ubuntu 18 specifically. Ref: http://hg.openjdk.java.net/jdk/jdk11/raw-file/tip/doc/building.html https://packages.debian.org/source/sid/openjdk-11 https://bugs.openjdk.java.net/browse/JDK-8205201 JDK-8144695 JDK-821204 --- thirdparty/openjdk/Makefile | 19 +- thirdparty/openjdk/UBUNTU.def | 24 ++ .../openjdk/patches/fix-JDK-8144695.patch | 11 + .../openjdk/patches/fix-JDK-8212041.patch | 228 ++++++++++++++++++ thirdparty/openjdk/patches/series | 2 + .../openjdk/zimbra-openjdk/debian/control | 12 +- .../zimbra-openjdk/debian/patches/series | 2 + .../openjdk/zimbra-openjdk/debian/rules | 9 +- versions.def | 10 +- 9 files changed, 286 insertions(+), 31 deletions(-) create mode 100644 thirdparty/openjdk/patches/fix-JDK-8144695.patch create mode 100644 thirdparty/openjdk/patches/fix-JDK-8212041.patch create mode 100644 thirdparty/openjdk/patches/series create mode 100644 thirdparty/openjdk/zimbra-openjdk/debian/patches/series diff --git a/thirdparty/openjdk/Makefile b/thirdparty/openjdk/Makefile index 5a5250864..5c5c0010f 100644 --- a/thirdparty/openjdk/Makefile +++ b/thirdparty/openjdk/Makefile @@ -25,23 +25,9 @@ all: clean getsrc build pkgrm1 getsrc: export HGFOREST_REDIRECT=getsource.out getsrc: mkdir -p $(SRC_DIR) - (echo "Obtaining source for openjdk. This may take some time." && \ + (echo "Copying the already created tar to src dir." && \ cd $(SRC_DIR) && \ - mkdir tmp && \ - cd tmp && \ - hg clone -r $(JDK_TAG) http://hg.openjdk.java.net/jdk8u/jdk$(JDK_SUB_TREE) && \ - cd jdk$(JDK_SUB_TREE) && chmod a+rx get_source.sh && \ - chmod a+rx configure && \ - patch -p1 < $(PKG_ROOT)/patches/fix-source.patch && \ - echo "Obtaining source for sub modules. This make take a very long time." && \ - ./get_source.sh -r $(JDK_TAG) || true && \ - cd .. && \ - mv jdk$(JDK_SUB_TREE) $(pname)-$(pvers) && \ - ( cd $(pname)-$(pvers) && find . -name ".hg*" | xargs rm -rf ) && \ - tar cfz $(pfile) $(pname)-$(pvers) && \ - mv $(pfile) .. && \ - cd .. && \ - rm -rf tmp \ + cp -vf $(PKG_ROOT)/$(pfile) . \ ) pkgadd: @@ -75,6 +61,7 @@ build_deb: perl -pi -e 's/DEB_BUILD_OPTIONS=nocheck/DEB_BUILD_OPTIONS=nocheck $(UB_REPLACE)/' debian/rules && \ $(replace-pathinfo) debian/rules && \ $(CP) $(psrc_file) ../$(z_tgz) && \ + $(CP) $(PKG_ROOT)/patches/*.patch debian/patches/ && \ $(TAR) xfz ../$(z_tgz) --strip-components=1 && \ $(PKG_BUILD) diff --git a/thirdparty/openjdk/UBUNTU.def b/thirdparty/openjdk/UBUNTU.def index 3e08ad46a..39ec59e46 100644 --- a/thirdparty/openjdk/UBUNTU.def +++ b/thirdparty/openjdk/UBUNTU.def @@ -3,11 +3,35 @@ openjdk.UBUNTU := libx11-dev x11proto-xext-dev libxext-dev \ x11proto-fixes-dev x11proto-record-dev libxtst-dev libxt-dev \ openjdk-7-jdk libcups2-dev libfreetype6-dev libasound2-dev \ ccache +openjdk.UBUNTU18_64 := openjdk-11-jdk at-spi2-core ca-certificates-java \ + libatk-bridge2.0-0 libatk-wrapper-java libatk-wrapper-java-jni \ + libatspi2.0-0 libfontenc1 libgtk-3-0 libgtk-3-bin libice-dev \ + libice6 libpthread-stubs0-dev libsm-dev libsm6 libx11-dev \ + libx11-doc libxau-dev libxaw7 libxcb1-dev libxdmcp-dev libxmu6 \ + libxmuu1 libxpm4 libxt-dev libxt6 libxv1 libxxf86dga1 \ + openjdk-11-jdk-headless openjdk-11-jre openjdk-11-jre-headless \ + x11-utils x11proto-core-dev x11proto-dev xorg-sgml-doctools \ + xtrans-dev libxext-dev libxrender-dev libxtst-dev libxaw7-dev \ + libxinerama-dev libgtk-3-dev libfontconfig1-dev xvfb xauth \ + xfonts-base twm x11-xkb-utils graphviz libcups2-dev \ + libfreetype6-dev libasound2-dev ccache jtreg ant ant-optional openjdk.UBUNTU16_64 := $(openjdk.UBUNTU) openjdk.UBUNTU14_64 := $(openjdk.UBUNTU) openjdk.UBUNTU12_64 := $(openjdk.UBUNTU) openjdk-purge.UBUNTU := libxt-dev libxtst-dev libxrender-dev libxext-dev libx11-dev \ libcups2-dev libfreetype6-dev libasound2-dev openjdk-7-jdk ccache +openjdk-purge.UBUNTU18_64 := openjdk-11-jdk at-spi2-core ca-certificates-java \ + libatk-bridge2.0-0 libatk-wrapper-java libatk-wrapper-java-jni \ + libatspi2.0-0 libfontenc1 libgtk-3-0 libgtk-3-bin libice-dev \ + libice6 libpthread-stubs0-dev libsm-dev libsm6 libx11-dev \ + libx11-doc libxau-dev libxaw7 libxcb1-dev libxdmcp-dev libxmu6 \ + libxmuu1 libxpm4 libxt-dev libxt6 libxv1 libxxf86dga1 \ + openjdk-11-jdk-headless openjdk-11-jre openjdk-11-jre-headless \ + x11-utils x11proto-core-dev x11proto-dev xorg-sgml-doctools \ + xtrans-dev libxext-dev libxrender-dev libxtst-dev libxaw7-dev \ + libxinerama-dev libgtk-3-dev libfontconfig1-dev xvfb xauth \ + xfonts-base twm x11-xkb-utils graphviz libcups2-dev \ + libfreetype6-dev libasound2-dev ccache jtreg ant ant-optional openjdk-purge.UBUNTU16_64 := $(openjdk-purge.UBUNTU) libsctp1 openjdk-7-jre-headless \ x11proto-fixes-dev x11proto-record-dev x11proto-render-dev x11proto-xext-dev \ x11proto-core-dev x11proto-input-dev xorg-sgml-doctools at-spi2-core \ diff --git a/thirdparty/openjdk/patches/fix-JDK-8144695.patch b/thirdparty/openjdk/patches/fix-JDK-8144695.patch new file mode 100644 index 000000000..e2fcffc48 --- /dev/null +++ b/thirdparty/openjdk/patches/fix-JDK-8144695.patch @@ -0,0 +1,11 @@ +--- a/src/hotspot/make/linux/makefiles/gcc.make ++++ b/src/hotspot/make/linux/makefiles/gcc.make +@@ -203,7 +203,7 @@ else + endif + + # Compiler warnings are treated as errors +-WARNINGS_ARE_ERRORS ?= -Werror -Wno-error=format ++WARNINGS_ARE_ERRORS ?= + + ifeq ($(USE_CLANG), true) + # However we need to clean the code up before we can unrestrictedly enable this option with Clang diff --git a/thirdparty/openjdk/patches/fix-JDK-8212041.patch b/thirdparty/openjdk/patches/fix-JDK-8212041.patch new file mode 100644 index 000000000..73c398c9d --- /dev/null +++ b/thirdparty/openjdk/patches/fix-JDK-8212041.patch @@ -0,0 +1,228 @@ +--- a/src/hotspot/os/linux/os_linux.cpp Tue Jul 17 12:03:10 2018 -0700 ++++ b/src/hotspot/os/linux/os_linux.cpp Tue Jul 17 15:59:47 2018 -0400 +@@ -5375,8 +5375,7 @@ + + // Scan the directory + bool result = true; +- char buf[sizeof(struct dirent) + MAX_PATH]; +- while (result && (ptr = ::readdir(dir)) != NULL) { ++ while (result && (ptr = readdir(dir)) != NULL) { + if (strcmp(ptr->d_name, ".") != 0 && strcmp(ptr->d_name, "..") != 0) { + result = false; + } +--- a/src/hotspot/os/linux/os_linux.inline.hpp Tue Jul 17 12:03:10 2018 -0700 ++++ b/src/hotspot/os/linux/os_linux.inline.hpp Tue Jul 17 15:59:47 2018 -0400 +@@ -69,17 +69,6 @@ + + inline const int os::default_file_open_flags() { return 0;} + +-inline DIR* os::opendir(const char* dirname) +-{ +- assert(dirname != NULL, "just checking"); +- return ::opendir(dirname); +-} +- +-inline int os::readdir_buf_size(const char *path) +-{ +- return NAME_MAX + sizeof(dirent) + 1; +-} +- + inline jlong os::lseek(int fd, jlong offset, int whence) { + return (jlong) ::lseek64(fd, offset, whence); + } +@@ -92,17 +81,6 @@ + return ::ftruncate64(fd, length); + } + +-inline struct dirent* os::readdir(DIR* dirp, dirent *dbuf) +-{ +- assert(dirp != NULL, "just checking"); +- return ::readdir(dirp); +-} +- +-inline int os::closedir(DIR *dirp) { +- assert(dirp != NULL, "argument is NULL"); +- return ::closedir(dirp); +-} +- + // macros for restartable system calls + + #define RESTARTABLE(_cmd, _result) do { \ +--- a/src/hotspot/os/linux/os_perf_linux.cpp Tue Jul 17 12:03:10 2018 -0700 ++++ b/src/hotspot/os/linux/os_perf_linux.cpp Tue Jul 17 15:59:47 2018 -0400 +@@ -895,21 +895,14 @@ + } + + int SystemProcessInterface::SystemProcesses::ProcessIterator::next_process() { +- struct dirent* entry; +- + if (!is_valid()) { + return OS_ERR; + } + + do { +- entry = os::readdir(_dir, _entry); +- if (entry == NULL) { +- // error +- _valid = false; +- return OS_ERR; +- } ++ _entry = os::readdir(_dir); + if (_entry == NULL) { +- // reached end ++ // Error or reached end. Could use errno to distinguish those cases. + _valid = false; + return OS_ERR; + } +@@ -926,11 +919,8 @@ + } + + bool SystemProcessInterface::SystemProcesses::ProcessIterator::initialize() { +- _dir = opendir("/proc"); +- _entry = (struct dirent*)NEW_C_HEAP_ARRAY(char, sizeof(struct dirent) + NAME_MAX + 1, mtInternal); +- if (NULL == _entry) { +- return false; +- } ++ _dir = os::opendir("/proc"); ++ _entry = NULL; + _valid = true; + next_process(); + +@@ -938,11 +928,8 @@ + } + + SystemProcessInterface::SystemProcesses::ProcessIterator::~ProcessIterator() { +- if (_entry != NULL) { +- FREE_C_HEAP_ARRAY(char, _entry); +- } + if (_dir != NULL) { +- closedir(_dir); ++ os::closedir(_dir); + } + } + +--- a/src/hotspot/os/linux/perfMemory_linux.cpp Tue Jul 17 12:03:10 2018 -0700 ++++ b/src/hotspot/os/linux/perfMemory_linux.cpp Tue Jul 17 15:59:47 2018 -0400 +@@ -561,9 +561,8 @@ + // to determine the user name for the process id. + // + struct dirent* dentry; +- char* tdbuf = NEW_C_HEAP_ARRAY(char, os::readdir_buf_size(tmpdirname), mtInternal); + errno = 0; +- while ((dentry = os::readdir(tmpdirp, (struct dirent *)tdbuf)) != NULL) { ++ while ((dentry = os::readdir(tmpdirp)) != NULL) { + + // check if the directory entry is a hsperfdata file + if (strncmp(dentry->d_name, PERFDATA_NAME, strlen(PERFDATA_NAME)) != 0) { +@@ -597,9 +596,8 @@ + } + + struct dirent* udentry; +- char* udbuf = NEW_C_HEAP_ARRAY(char, os::readdir_buf_size(usrdir_name), mtInternal); + errno = 0; +- while ((udentry = os::readdir(subdirp, (struct dirent *)udbuf)) != NULL) { ++ while ((udentry = os::readdir(subdirp)) != NULL) { + + if (filename_to_pid(udentry->d_name) == searchpid) { + struct stat statbuf; +@@ -643,11 +641,9 @@ + } + } + os::closedir(subdirp); +- FREE_C_HEAP_ARRAY(char, udbuf); + FREE_C_HEAP_ARRAY(char, usrdir_name); + } + os::closedir(tmpdirp); +- FREE_C_HEAP_ARRAY(char, tdbuf); + + return(oldest_user); + } +@@ -769,10 +765,8 @@ + // opendir/readdir. + // + struct dirent* entry; +- char* dbuf = NEW_C_HEAP_ARRAY(char, os::readdir_buf_size(dirname), mtInternal); +- + errno = 0; +- while ((entry = os::readdir(dirp, (struct dirent *)dbuf)) != NULL) { ++ while ((entry = os::readdir(dirp)) != NULL) { + + pid_t pid = filename_to_pid(entry->d_name); + +@@ -809,8 +803,6 @@ + + // close the directory and reset the current working directory + close_directory_secure_cwd(dirp, saved_cwd_fd); +- +- FREE_C_HEAP_ARRAY(char, dbuf); + } + + // make the user specific temporary directory. Returns true if + +--- a/src/hotspot/os/posix/os_posix.cpp Tue Jul 17 12:03:10 2018 -0700 ++++ b/src/hotspot/os/posix/os_posix.cpp Tue Jul 17 15:59:47 2018 -0400 +@@ -35,6 +35,7 @@ + #include "utilities/macros.hpp" + #include "utilities/vmError.hpp" + ++#include + #include + #include + #include +@@ -527,6 +528,21 @@ + ::funlockfile(fp); + } + ++DIR* os::opendir(const char* dirname) { ++ assert(dirname != NULL, "just checking"); ++ return ::opendir(dirname); ++} ++ ++struct dirent* os::readdir(DIR* dirp) { ++ assert(dirp != NULL, "just checking"); ++ return ::readdir(dirp); ++} ++ ++int os::closedir(DIR *dirp) { ++ assert(dirp != NULL, "just checking"); ++ return ::closedir(dirp); ++} ++ + // Builds a platform dependent Agent_OnLoad_ function name + // which is used to find statically linked in agents. + // Parameters: + +--- a/src/hotspot/share/jfr/recorder/repository/jfrRepository.cpp Tue Jul 17 12:03:10 2018 -0700 ++++ b/src/hotspot/share/jfr/recorder/repository/jfrRepository.cpp Tue Jul 17 15:59:47 2018 -0400 +@@ -241,11 +241,7 @@ + return; + } + struct dirent* dentry; +- char* dir_buffer = NEW_RESOURCE_ARRAY_RETURN_NULL(char, os::readdir_buf_size(_repo)); +- if (dir_buffer == NULL) { +- return; +- } +- while ((dentry = os::readdir(dirp, (struct dirent*)dir_buffer)) != NULL) { ++ while ((dentry = os::readdir(dirp)) != NULL) { + const char* const entry_path = filter(dentry->d_name); + if (NULL != entry_path) { + _files->append(entry_path); +--- a/src/hotspot/share/runtime/os.hpp Tue Jul 17 12:03:10 2018 -0700 ++++ b/src/hotspot/share/runtime/os.hpp Tue Jul 17 15:59:47 2018 -0400 +@@ -580,8 +580,7 @@ + + // Reading directories. + static DIR* opendir(const char* dirname); +- static int readdir_buf_size(const char *path); +- static struct dirent* readdir(DIR* dirp, dirent* dbuf); ++ static struct dirent* readdir(DIR* dirp); + static int closedir(DIR* dirp); + + // Dynamic library extension +--- a/test/jdk/ProblemList.txt Tue Jul 17 12:03:10 2018 -0700 ++++ b/test/jdk/ProblemList.txt Tue Jul 17 15:59:47 2018 -0400 +@@ -873,4 +873,3 @@ + + jdk/jfr/event/io/TestInstrumentation.java 8202142 generic-all + jdk/jfr/event/sampling/TestNative.java 8202142 generic-all +-jdk/jfr/event/os/TestSystemProcess.java 8202835 linux-all diff --git a/thirdparty/openjdk/patches/series b/thirdparty/openjdk/patches/series new file mode 100644 index 000000000..a708dc8e0 --- /dev/null +++ b/thirdparty/openjdk/patches/series @@ -0,0 +1,2 @@ +fix-JDK-8144695.patch +fix-JDK-8212041.patch diff --git a/thirdparty/openjdk/zimbra-openjdk/debian/control b/thirdparty/openjdk/zimbra-openjdk/debian/control index a3048a979..8bab44a84 100644 --- a/thirdparty/openjdk/zimbra-openjdk/debian/control +++ b/thirdparty/openjdk/zimbra-openjdk/debian/control @@ -1,7 +1,13 @@ Source: zimbra-openjdk -Build-Depends: debhelper (>= 9), m4, dpkg-dev (>= 1.15.7), libx11-dev, - libxext-dev, libxrender-dev, libxtst-dev, libxt-dev, openjdk-7-jdk, - libcups2-dev, libfreetype6-dev, libasound2-dev, ccache, zip +Build-Depends: debhelper (>= 9.20141010), dpkg-dev (>= 1.17.14), m4, lsb-release, zip, + unzip, sharutils, gawk, cpio, pkg-config, procps, wdiff, fastjar (>=2:0.96-0ubuntu2), + autoconf, automake, autotools-dev, ant, ant-optional, g++-8, + libxtst-dev, libxi-dev, libxt-dev, libxaw7-dev, libxrender-dev, libcups2-dev, + libasound2-dev, liblcms2-dev, libfreetype6-dev (>= 2.2.1), libxinerama-dev, + libkrb5-dev, xsltproc, libpcsclite-dev, libgtk-3-dev, libelf-dev, libfontconfig1-dev, + libffi-dev, zlib1g-dev, libattr1-dev, libpng-dev, libjpeg-dev, libgif-dev, libnss3-dev, + jtreg (>=4.2-b10-1~), xvfb, xauth, xfonts-base, libgl1-mesa-dri, twm, x11-xkb-utils, + testng, time, openjdk-11-jdk-headless, graphviz, pandoc Section: utils Priority: optional Maintainer: Zimbra Packaging Services diff --git a/thirdparty/openjdk/zimbra-openjdk/debian/patches/series b/thirdparty/openjdk/zimbra-openjdk/debian/patches/series new file mode 100644 index 000000000..a708dc8e0 --- /dev/null +++ b/thirdparty/openjdk/zimbra-openjdk/debian/patches/series @@ -0,0 +1,2 @@ +fix-JDK-8144695.patch +fix-JDK-8212041.patch diff --git a/thirdparty/openjdk/zimbra-openjdk/debian/rules b/thirdparty/openjdk/zimbra-openjdk/debian/rules index b6fb856c4..9ed5ae397 100755 --- a/thirdparty/openjdk/zimbra-openjdk/debian/rules +++ b/thirdparty/openjdk/zimbra-openjdk/debian/rules @@ -9,9 +9,8 @@ override_dh_auto_clean: override_dh_auto_configure: ./configure --prefix=OZC \ --enable-unlimited-crypto \ - --with-update-version=JDK_UPDATE \ - --with-build-number=JDK_BUILD \ - --with-milestone=zimbra + --disable-warnings-as-errors \ + --enable-debug override_dh_strip: dh_strip -pzimbra-openjdk --dbg-package=zimbra-openjdk-dbg @@ -26,14 +25,10 @@ override_dh_auto_install: cd $(CURDIR)/debian/tmpOZCL/jvm && ln -s openjdk* java rm -rf $(CURDIR)/debian/tmp/OZCL/jvm/java/demo rm -rf $(CURDIR)/debian/tmp/OZCL/jvm/java/sample - rm -f $(CURDIR)/debian/tmp/OZCL/jvm/java/jre/lib/security/cacerts - cd $(CURDIR)/debian/tmp/OZCL/jvm/java/jre/lib/security && \ - ln -s OZCE/java/cacerts cacerts cd $(CURDIR)/debian/tmpOZCB && ln -s ../lib/jvm/java/bin/jar cd $(CURDIR)/debian/tmpOZCB && ln -s ../lib/jvm/java/bin/java cd $(CURDIR)/debian/tmpOZCB && ln -s ../lib/jvm/java/bin/javac cd $(CURDIR)/debian/tmpOZCB && ln -s ../lib/jvm/java/bin/javap - cd $(CURDIR)/debian/tmpOZCB && ln -s ../lib/jvm/java/bin/jhat cd $(CURDIR)/debian/tmpOZCB && ln -s ../lib/jvm/java/bin/jmap cd $(CURDIR)/debian/tmpOZCB && ln -s ../lib/jvm/java/bin/jps cd $(CURDIR)/debian/tmpOZCB && ln -s ../lib/jvm/java/bin/jstack diff --git a/versions.def b/versions.def index 6e9ea5711..85056f991 100644 --- a/versions.def +++ b/versions.def @@ -69,11 +69,11 @@ HTTPD_VERSION := 2.4.20 IMGK_VERSION := 6.7.7-2 -JDK_SUB_TREE := 8u -JDK_UPDATE := 172 -JDK_BUILD := b01 -JDK_TAG := jdk8u$(JDK_UPDATE)-$(JDK_BUILD) -JDK_VERSION := 1.8.0u$(JDK_UPDATE)$(JDK_BUILD) +JDK_SUB_TREE := 11 +JDK_UPDATE := 11 +JDK_BUILD := 28 +JDK_TAG := jdk-$(JDK_SUB_TREE)+$(JDK_BUILD) +JDK_VERSION := $(JDK_UPDATE).$(JDK_BUILD) JETTY_DISTRIBUTION_VERSION := 9.3.5.v20151012 From 92e81d9c365f41e5da69c627d64f71bdd3fffcea Mon Sep 17 00:00:00 2001 From: Prashant Surana Date: Mon, 12 Nov 2018 11:22:56 +0000 Subject: [PATCH 14/15] ZCS-6172: Remove openjdk-11.28.tgz from the commit as this is the file larger then 100 mb --- thirdparty/openjdk/.gitattributes | 1 + 1 file changed, 1 insertion(+) create mode 100644 thirdparty/openjdk/.gitattributes diff --git a/thirdparty/openjdk/.gitattributes b/thirdparty/openjdk/.gitattributes new file mode 100644 index 000000000..4c7f77287 --- /dev/null +++ b/thirdparty/openjdk/.gitattributes @@ -0,0 +1 @@ +openjdk-11.28.tgz filter=lfs diff=lfs merge=lfs -text From e2ff3492b43e7c2726f9e1c278cf2b84bf52d41b Mon Sep 17 00:00:00 2001 From: Amit Srivastava Date: Thu, 6 Dec 2018 18:09:40 +0000 Subject: [PATCH 15/15] Ubuntu18 releated specific changes --- .gitignore | 2 ++ thirdparty/bdb/Makefile | 2 +- thirdparty/freetype/Makefile | 8 ++++---- thirdparty/mariadb/Makefile | 3 ++- thirdparty/perl-config-inifiles/UBUNTU.def | 1 + thirdparty/perl-dbd-mysql/Makefile | 4 ++-- thirdparty/perl-encode-detect/UBUNTU.def | 1 + thirdparty/perl-mail-dkim/Makefile | 4 ++-- .../perl-mail-dkim/zimbra-perl-mail-dkim/debian/rules | 2 +- thirdparty/perl-mail-spf/UBUNTU.def | 1 + thirdparty/perl-net-dns-resolver-programmable/UBUNTU.def | 1 + thirdparty/perl-net-server/Makefile | 2 +- thirdparty/perl-xml-simple/Makefile | 2 +- .../perl-xml-simple/zimbra-perl-xml-simple/debian/rules | 3 ++- thirdparty/perl-zmq-libzmq3/UBUNTU.def | 1 + versions.def | 2 +- zimbra/os-requirements/UBUNTU.def | 1 + 17 files changed, 25 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index 567609b12..8499fa922 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ build/ +thirdparty/openjdk/openjdk-11.28.tgz +thirdparty/openjdk/make_openjdk11.log diff --git a/thirdparty/bdb/Makefile b/thirdparty/bdb/Makefile index e273939b3..6256e2b75 100644 --- a/thirdparty/bdb/Makefile +++ b/thirdparty/bdb/Makefile @@ -7,7 +7,7 @@ pname := bdb uname := db pfile := $(uname)-$(pvers).tar.gz psrc_file := $(SRC_DIR)/$(pfile) -purl := http://ftp.nsysu.edu.tw/FreeBSD/ports/distfiles/bdb/$(pfile) +purl := https://ftp.tw.freebsd.org/distfiles/bdb/$(pfile) zname := zimbra-$(pname) zspec := $(pname).spec diff --git a/thirdparty/freetype/Makefile b/thirdparty/freetype/Makefile index e480cf8f5..b2c30d4b5 100644 --- a/thirdparty/freetype/Makefile +++ b/thirdparty/freetype/Makefile @@ -4,9 +4,9 @@ include $(PKG_ROOT)/../../package.def pvers := $(FREETYPE_VERSION) pname := freetype -pfile := $(pname)-$(pvers).tar.gz +pfile := $(pname)-$(pvers).tar.bz2 psrc_file := $(SRC_DIR)/$(pfile) -purl := http://download.savannah.gnu.org/releases/freetype/$(pfile) +purl := https://oss.oetiker.ch/rrdtool/pub/libs/$(pfile) zname := zimbra-$(pname) zspec := $(pname).spec @@ -33,14 +33,14 @@ build_rpm: $(CP) $(psrc_file) SOURCES/$(zname)-$(pvers).tar.gz && \ $(PKG_BUILD) $(specfile) -build_deb: z_tgz = $(zname)_$(pvers).orig.tar.gz +build_deb: z_tgz = $(zname)_$(pvers).orig.tar.bz2 build_deb: z_shlibs = $(subst $(zname)/,,$(wildcard $(zname)/debian/z*.shlibs)) build_deb: $(CD) $(PLATFORM_DIR)/$(zname) && \ $(replace-pkginfo) debian/changelog $(z_shlibs) && \ $(replace-pathinfo) debian/rules && \ $(CP) $(psrc_file) ../$(z_tgz) && \ - $(TAR) xfz ../$(z_tgz) --strip-components=1 && \ + $(TAR) xfj ../$(z_tgz) --strip-components=1 && \ $(PKG_BUILD) clean: diff --git a/thirdparty/mariadb/Makefile b/thirdparty/mariadb/Makefile index d1eca0287..af56198ae 100644 --- a/thirdparty/mariadb/Makefile +++ b/thirdparty/mariadb/Makefile @@ -6,7 +6,8 @@ pvers := $(MARIADB_VERSION) pname := mariadb pfile := $(pname)-$(pvers).tar.gz psrc_file := $(SRC_DIR)/$(pfile) -purl := https://downloads.mariadb.org/f/$(pname)-$(pvers)/source/$(pfile) +#purl := https://downloads.mariadb.org/f/$(pname)-$(pvers)/source/$(pfile) +purl := http://ftp.hosteurope.de/mirror/archive.mariadb.org//$(pname)-$(pvers)/source/$(pfile) zname := zimbra-$(pname) zspec := $(pname).spec diff --git a/thirdparty/perl-config-inifiles/UBUNTU.def b/thirdparty/perl-config-inifiles/UBUNTU.def index 60f239654..0c7d99fc8 100644 --- a/thirdparty/perl-config-inifiles/UBUNTU.def +++ b/thirdparty/perl-config-inifiles/UBUNTU.def @@ -1,4 +1,5 @@ module-build.UBUNTU := +module-build.UBUNTU18_64 := libmodule-build-perl module-build.UBUNTU16_64 := libmodule-build-perl module-build.UBUNTU14_64 := $(module-build.UBUNTU) module-build.UBUNTU12_64 := $(module-build.UBUNTU) diff --git a/thirdparty/perl-dbd-mysql/Makefile b/thirdparty/perl-dbd-mysql/Makefile index e690d1eac..82a35bc55 100644 --- a/thirdparty/perl-dbd-mysql/Makefile +++ b/thirdparty/perl-dbd-mysql/Makefile @@ -2,13 +2,13 @@ PKG_ROOT := $(shell pwd) include $(PKG_ROOT)/../../package.def -pvers := $(DBD_MYSQL) +pvers := 4.043 pname := DBD-mysql pname_lc := dbd-mysql pfile := $(pname)-$(pvers).tar.gz psrc_file := $(SRC_DIR)/$(pfile) -purl := https://cpan.metacpan.org/authors/id/M/MI/MICHIELB/$(pfile) +purl := https://www.cpan.org/modules/by-module/DBD/$(pfile) zname := zimbra-perl-$(pname_lc) zspec := $(pname_lc).spec diff --git a/thirdparty/perl-encode-detect/UBUNTU.def b/thirdparty/perl-encode-detect/UBUNTU.def index 60f239654..0c7d99fc8 100644 --- a/thirdparty/perl-encode-detect/UBUNTU.def +++ b/thirdparty/perl-encode-detect/UBUNTU.def @@ -1,4 +1,5 @@ module-build.UBUNTU := +module-build.UBUNTU18_64 := libmodule-build-perl module-build.UBUNTU16_64 := libmodule-build-perl module-build.UBUNTU14_64 := $(module-build.UBUNTU) module-build.UBUNTU12_64 := $(module-build.UBUNTU) diff --git a/thirdparty/perl-mail-dkim/Makefile b/thirdparty/perl-mail-dkim/Makefile index a6a852e20..e52214b3c 100644 --- a/thirdparty/perl-mail-dkim/Makefile +++ b/thirdparty/perl-mail-dkim/Makefile @@ -2,13 +2,13 @@ PKG_ROOT := $(shell pwd) include $(PKG_ROOT)/../../package.def -pvers := $(MAIL_DKIM) +pvers := 0.43 pname := Mail-DKIM pname_lc := mail-dkim pfile := $(pname)-$(pvers).tar.gz psrc_file := $(SRC_DIR)/$(pfile) -purl := https://cpan.metacpan.org/authors/id/J/JA/JASLONG/$(pfile) +purl := https://www.cpan.org/modules/by-module/Mail/$(pfile) zname := zimbra-perl-$(pname_lc) zspec := $(pname_lc).spec diff --git a/thirdparty/perl-mail-dkim/zimbra-perl-mail-dkim/debian/rules b/thirdparty/perl-mail-dkim/zimbra-perl-mail-dkim/debian/rules index fe2fa08d7..0cfcb8c45 100644 --- a/thirdparty/perl-mail-dkim/zimbra-perl-mail-dkim/debian/rules +++ b/thirdparty/perl-mail-dkim/zimbra-perl-mail-dkim/debian/rules @@ -6,7 +6,7 @@ override_dh_auto_configure: perl -I OZCL/perl5 Makefile.PL INSTALL_BASE=OZC \ INSTALLSITEMAN1DIR=OZCS/man/man1 INSTALLSITEMAN3DIR=OZCS/man/man3 \ - LIB=OZCL/perl5 + LIB=OZCL/perl5,OZCL/x86_64-linux-gnu-thread-multi # these need to know about our custom location for modules override_dh_perl: diff --git a/thirdparty/perl-mail-spf/UBUNTU.def b/thirdparty/perl-mail-spf/UBUNTU.def index 60f239654..0c7d99fc8 100644 --- a/thirdparty/perl-mail-spf/UBUNTU.def +++ b/thirdparty/perl-mail-spf/UBUNTU.def @@ -1,4 +1,5 @@ module-build.UBUNTU := +module-build.UBUNTU18_64 := libmodule-build-perl module-build.UBUNTU16_64 := libmodule-build-perl module-build.UBUNTU14_64 := $(module-build.UBUNTU) module-build.UBUNTU12_64 := $(module-build.UBUNTU) diff --git a/thirdparty/perl-net-dns-resolver-programmable/UBUNTU.def b/thirdparty/perl-net-dns-resolver-programmable/UBUNTU.def index 60f239654..0c7d99fc8 100644 --- a/thirdparty/perl-net-dns-resolver-programmable/UBUNTU.def +++ b/thirdparty/perl-net-dns-resolver-programmable/UBUNTU.def @@ -1,4 +1,5 @@ module-build.UBUNTU := +module-build.UBUNTU18_64 := libmodule-build-perl module-build.UBUNTU16_64 := libmodule-build-perl module-build.UBUNTU14_64 := $(module-build.UBUNTU) module-build.UBUNTU12_64 := $(module-build.UBUNTU) diff --git a/thirdparty/perl-net-server/Makefile b/thirdparty/perl-net-server/Makefile index e8df02c2e..b267b59cc 100644 --- a/thirdparty/perl-net-server/Makefile +++ b/thirdparty/perl-net-server/Makefile @@ -2,7 +2,7 @@ PKG_ROOT := $(shell pwd) include $(PKG_ROOT)/../../package.def -pvers := $(NET_SERVER) +pvers := 2.009 pname := Net-Server pname_lc := net-server diff --git a/thirdparty/perl-xml-simple/Makefile b/thirdparty/perl-xml-simple/Makefile index 8db57c210..78f18d6ee 100644 --- a/thirdparty/perl-xml-simple/Makefile +++ b/thirdparty/perl-xml-simple/Makefile @@ -2,7 +2,7 @@ PKG_ROOT := $(shell pwd) include $(PKG_ROOT)/../../package.def -pvers := $(XML_SIMPLE) +pvers := 2.25 pname := XML-Simple pname_lc := xml-simple diff --git a/thirdparty/perl-xml-simple/zimbra-perl-xml-simple/debian/rules b/thirdparty/perl-xml-simple/zimbra-perl-xml-simple/debian/rules index 70e9b1e61..588995b15 100644 --- a/thirdparty/perl-xml-simple/zimbra-perl-xml-simple/debian/rules +++ b/thirdparty/perl-xml-simple/zimbra-perl-xml-simple/debian/rules @@ -1,12 +1,13 @@ #!/usr/bin/make -f + %: dh $@ override_dh_auto_configure: perl -I OZCL/perl5 Makefile.PL INSTALL_BASE=OZC \ INSTALLSITEMAN1DIR=OZCS/man/man1 INSTALLSITEMAN3DIR=OZCS/man/man3 \ - LIB=OZCL/perl5 + LIB=OZCL/perl5,OZCL/perl5/x86_64-linux-gnu-thread-multi # these need to know about our custom location for modules override_dh_perl: diff --git a/thirdparty/perl-zmq-libzmq3/UBUNTU.def b/thirdparty/perl-zmq-libzmq3/UBUNTU.def index d1a8f577a..4f5f67d23 100644 --- a/thirdparty/perl-zmq-libzmq3/UBUNTU.def +++ b/thirdparty/perl-zmq-libzmq3/UBUNTU.def @@ -1,4 +1,5 @@ zptest.UBUNTU := libtest-fatal-perl libtest-requires-perl libtest-sharedfork-perl libtry-tiny-perl +zptest.UBUNTU18_64 := $(zptest.UBUNTU) zptest.UBUNTU16_64 := $(zptest.UBUNTU) zptest.UBUNTU14_64 := $(zptest.UBUNTU) zptest.UBUNTU12_64 := $(zptest.UBUNTU) diff --git a/versions.def b/versions.def index 85056f991..af836174c 100644 --- a/versions.def +++ b/versions.def @@ -55,7 +55,7 @@ CYRUS_VERSION := 2.1.26 DKIM_VERSION := 2.10.3 -FREETYPE_VERSION := 2.6.3 +FREETYPE_VERSION := 2.1.10 GPG_VERSION := 2.0.20 GPGERROR_VERSION := 1.12 diff --git a/zimbra/os-requirements/UBUNTU.def b/zimbra/os-requirements/UBUNTU.def index 4eb5c6ec2..d27d846d2 100644 --- a/zimbra/os-requirements/UBUNTU.def +++ b/zimbra/os-requirements/UBUNTU.def @@ -1,3 +1,4 @@ +osdeps.UBUNTU18_64 := libgmp10, libperl5.26 osdeps.UBUNTU16_64 := libgmp10, libperl5.22 osdeps.UBUNTU14_64 := libgmp10, libperl5.18 osdeps.UBUNTU12_64 := libgmp3c2, libperl5.14