diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index cdc0aa42843..75a8fc5c6fe 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -5,7 +5,7 @@ name: Frontend CI on: push: - branches: [master] + branches: ["*"] paths: - "src/frontend/**" pull_request: diff --git a/README.md b/README.md index c79e7dca228..08afad6fb6a 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,8 @@ bk-ci提供了流水线、代码检查、代码库、凭证管理、环境管理 - [BK-JOB](https://github.com/Tencent/bk-job)：蓝鲸作业平台(Job)是一套运维脚本管理系统，具备海量任务并发处理能力。 - [BK-PaaS](https://github.com/Tencent/bk-PaaS)：蓝鲸PaaS平台是一个开放式的开发平台，让开发者可以方便快捷地创建、开发、部署和管理SaaS应用。 - [BK-SOPS](https://github.com/Tencent/bk-sops)：蓝鲸标准运维（SOPS）是通过可视化的图形界面进行任务流程编排和执行的系统，是蓝鲸体系中一款轻量级的调度编排类SaaS产品。 +- [BK-Repo](https://github.com/Tencentblueking/bk-repo)：蓝鲸制品库平台是一套为企业提供各种类型制品包存储、代理、分发、晋级、扫描、依赖包管理的持续交付平台。 +- [BK-Turbo](https://github.com/Tencentblueking/bk-turbo): 蓝鲸编译加速平台为CI场景下提供UE、C/C++等多种语言的编译加速服务能力 ## Contributing - 关于 bk-ci 分支管理、issue 以及 pr 规范，请阅读 [Contributing](CONTRIBUTING.md) diff --git a/README_EN.md b/README_EN.md index 6d6b0727b05..05e05a0b02e 100644 --- a/README_EN.md +++ b/README_EN.md @@ -61,7 +61,9 @@ bk-ci provides seven core services, namely Process, CodeCheck, Repository, Ticke - [BK-CMDB](https://github.com/Tencent/bk-cmdb): BlueKing Configuration Management DataBase (BlueKing CMDB) is an enterprise level configuration management platform for assets and applications. - [BK-JOB](https://github.com/Tencent/bk-job): BlueKing JOB is a set of operation and maintenance script management platform with the ability to handle a large number of tasks concurrently. - [BK-PaaS](https://github.com/Tencent/bk-PaaS): BlueKing PaaS is an open development platform that allows developers to create, develop, deploy and manage SaaS applications quickly and easily. +- [BK-Repo](https://github.com/Tencentblueking/bk-repo): The BlueKing Artifact Repository Platform is a continuous delivery platform that provides enterprises with various types of artifact package storage, proxy, distribution, promotion, scanning, and dependency package management. - [BK-SOPS](https://github.com/Tencent/bk-sops): BlueKing Standard OPS (SOPS) is a light-weighted SaaS product in the Tencent BlueKing product system designed for the orchestration and execution of tasks through a graphical interface. +- [BK-Turbo](https://github.com/Tencentblueking/bk-turbo): The BlueKing Turbo Platform provides compilation acceleration services for various languages, including UE and C/C++, in CI scenarios. ## Contributing - Please read [Contributing](CONTRIBUTING.en.md) for the branch management, issue and pr specifications of bk-ci. diff --git a/docs/overview/db/data_clear.md b/docs/overview/db/data_clear.md new file mode 100644 index 00000000000..96c63b9905b --- /dev/null +++ b/docs/overview/db/data_clear.md @@ -0,0 +1,141 @@ +# 蓝盾数据清理 +## 背景 + + 随着蓝盾的构建量快速增长， 蓝盾DB的数据量也越来越大，而蓝盾的DB的容量是有限的，故蓝盾需要一套DB数据清理方案来保证系统的稳定。 + + + +## 清理步骤 + +#### 一、确定表数据清理方案 + +​ 蓝盾的表可以分为流水数据表和非流水线数据表这二种类型，流水数据表很早之前的数据用户并不关心，所以流水线数库表可以通过分区表的方式来定时清理数据；非流水线数据表的数据相对比较重要，没法简单按时间维度删除数据，故我们需要开发一个依据特定条件删除数据的定时任务。 + + + +#### 二、分区表数据清理 + +##### 1、确定数据库表是否能调整为分区表 + +**新增表：** 新表没有历史包袱，只需明确分区的字段和每个分区的大小就行，以下是按时间就行分区表的建表语句： + +``` +CREATE TABLE `T_REPOSITORY_COMMIT` ( + `ID` bigint(20) NOT NULL AUTO_INCREMENT, + `BUILD_ID` varchar(34) DEFAULT NULL, + `PIPELINE_ID` varchar(34) DEFAULT NULL, + `REPO_ID` bigint(20) DEFAULT NULL, + `TYPE` smallint(6) DEFAULT NULL COMMENT '1-svn, 2-git, 3-gitlab', + `COMMIT` varchar(64) DEFAULT NULL, + `COMMITTER` varchar(64) DEFAULT NULL, + `COMMIT_TIME` datetime DEFAULT NULL, + `COMMENT` longtext /*!99104 COMPRESSED */, + `ELEMENT_ID` varchar(34) DEFAULT NULL, + `REPO_NAME` varchar(128) DEFAULT NULL, + `CREATE_TIME` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `URL` varchar(255) DEFAULT NULL COMMENT '代码库URL', + PRIMARY KEY (`ID`,`CREATE_TIME`), + KEY `BUILD_ID_INDEX` (`BUILD_ID`), + KEY `IDX_PIPE_ELEMENT_REPO_TIME` (`PIPELINE_ID`,`ELEMENT_ID`,`REPO_ID`,`COMMIT_TIME`), + KEY `IDX_BUILD_ID_TIME` (`BUILD_ID`,`COMMIT_TIME`), + KEY `IDX_PIPE_ELEMENT_NAME_REPO_TIME` (`PIPELINE_ID`,`ELEMENT_ID`,`REPO_NAME`,`COMMIT_TIME`) +) ENGINE=InnoDB AUTO_INCREMENT=3170191500 DEFAULT CHARSET=utf8mb4 +/*!50100 PARTITION BY RANGE (TO_DAYS(CREATE_TIME)) +(PARTITION p20240424 VALUES LESS THAN (739366) ENGINE = InnoDB, + PARTITION p20240501 VALUES LESS THAN (739373) ENGINE = InnoDB, + PARTITION p20240508 VALUES LESS THAN (739380) ENGINE = InnoDB, + PARTITION p20240515 VALUES LESS THAN (739387) ENGINE = InnoDB, + PARTITION p20240522 VALUES LESS THAN (739394) ENGINE = InnoDB, + PARTITION p20240529 VALUES LESS THAN (739401) ENGINE = InnoDB, + PARTITION p20240605 VALUES LESS THAN (739408) ENGINE = InnoDB, + PARTITION p20240612 VALUES LESS THAN (739415) ENGINE = InnoDB, + PARTITION p20240619 VALUES LESS THAN (739422) ENGINE = InnoDB, + PARTITION p20240626 VALUES LESS THAN (739429) ENGINE = InnoDB, + PARTITION p20240703 VALUES LESS THAN (739436) ENGINE = InnoDB, + PARTITION p20240710 VALUES LESS THAN (739443) ENGINE = InnoDB, + PARTITION p20240717 VALUES LESS THAN (739450) ENGINE = InnoDB, + PARTITION p20240724 VALUES LESS THAN (739457) ENGINE = InnoDB, + PARTITION p20240731 VALUES LESS THAN (739464) ENGINE = InnoDB, + PARTITION p20240807 VALUES LESS THAN (739471) ENGINE = InnoDB, + PARTITION p20240814 VALUES LESS THAN (739478) ENGINE = InnoDB, + PARTITION p20240821 VALUES LESS THAN (739485) ENGINE = InnoDB, + PARTITION p20240828 VALUES LESS THAN (739492) ENGINE = InnoDB) */; +``` + +**存量表：** 存量表要在不影响现有业务的情况下调整为分区表相对比较麻烦点，以T_REPOSITORY_COMMIT表为例，以下是具体步骤： + +1、创建临时表T_REPOSITORY_COMMIT_TMP(必须为分区表，分区键必须包含在主键内) + +2、把T_REPOSITORY_COMMIT表的数据同步至T_REPOSITORY_COMMIT_TMP表并建立同步方案 + +3、执行 RENAME TABLE T_REPOSITORY_COMMIT TO T_REPOSITORY_COMMIT_BAK,T_REPOSITORY_COMMIT_TMP TO T_REPOSITORY_COMMIT ,T_REPOSITORY_COMMIT_BAK TO T_REPOSITORY_COMMIT_TMP; 该语句进行新旧表名互换。 + +4、删除T_REPOSITORY_COMMIT_TMP表 + + + +##### 2、为分区表创立自动创建分区和删除分区的执行计划 + + + +**各微服务数据库的分区表情况：** + +| 数据库名称 | 表名 | 分区字段 | 过期时间（单位：天） | 分区间隔（单位：天） | +| ----------------- | ----------------------------------- | --------------- | -------------------- | -------------------- | +| devops_process | T_PIPELINE_TRIGGER_EVENT | CREATE_TIME | 35 | 7 | +| devops_process | T_PIPELINE_TRIGGER_DETAIL | CREATE_TIME | 35 | 7 | +| devops_process | T_PIPELINE_BUILD_VAR | CREATE_TIME | 42 | 7 | +| devops_process | T_PIPELINE_BUILD_TASK | CREATE_TIME | 42 | 7 | +| devops_process | T_PIPELINE_BUILD_STAGE | CREATE_TIME | 42 | 7 | +| devops_process | T_PIPELINE_BUILD_CONTAINER | CREATE_TIME | 42 | 7 | +| devops_process | T_PROJECT_PIPELINE_CALLBACK_HISTORY | CREATE_TIME | 3 | 3 | +| devops_process | T_PIPELINE_WEBHOOK_BUILD_LOG | CREATE_TIME | 3 | 3 | +| devops_process | T_PIPELINE_WEBHOOK_BUILD_LOG_DETAIL | CREATE_TIME | 3 | 3 | +| devops_repository | T_REPOSITORY_COMMIT | CREATE_TIME | 35 | 7 | +| devops_repository | T_REPOSITORY_WEBHOOK_REQUEST | CREATE_TIME | 35 | 7 | +| devops_notify | T_NOTIFY_WEWORK | CREATED_TIME | 8 | 1 | +| devops_notify | T_NOTIFY_EMAIL | CREATED_TIME | 8 | 1 | +| devops_notify | T_NOTIFY_RTX | CREATED_TIME | 8 | 1 | +| devops_notify | T_NOTIFY_SMS | CREATED_TIME | 8 | 1 | +| devops_notify | T_NOTIFY_WECHAT | CREATED_TIME | 8 | 1 | +| devops_plugin | T_PLUGIN_GIT_CHECK | CREATE_TIME | 31 | 1 | +| devops_stream | T_GIT_REQUEST_EVENT_BUILD | CREATE_TIME | 35 | 7 | +| devops_stream | T_GIT_REQUEST_REPO_EVENT | CREATE_TIME | 35 | 7 | +| devops_stream | T_GIT_REQUEST_EVENT_NOT_BUILD | CREATE_TIME | 35 | 7 | +| devops_stream | T_GIT_USER_MESSAGE | CREATE_TIME | 35 | 7 | +| devops_stream | T_GIT_REQUEST_EVENT | CREATE_TIME | 35 | 7 | +| devops_metrics | T_PROJECT_THIRD_PLATFORM_DATA | CREATE_TIME | 371 | 7 | +| devops_metrics | T_PIPELINE_FAIL_DETAIL_DATA | CREATE_TIME | 371 | 7 | +| devops_metrics | T_PIPELINE_FAIL_SUMMARY_DATA | CREATE_TIME | 371 | 7 | +| devops_metrics | T_ATOM_MONITOR_DATA_DAILY | STATISTICS_TIME | 371 | 7 | +| devops_metrics | T_ATOM_INDEX_STATISTICS_DAILY | CREATE_TIME | 371 | 7 | +| devops_metrics | T_ATOM_FAIL_DETAIL_DATA | CREATE_TIME | 210 | 7 | +| devops_metrics | T_PIPELINE_OVERVIEW_DATA | CREATE_TIME | 371 | 7 | +| devops_metrics | T_ATOM_FAIL_SUMMARY_DATA | CREATE_TIME | 371 | 7 | +| devops_metrics | T_PIPELINE_STAGE_OVERVIEW_DATA | CREATE_TIME | 371 | 7 | +| devops_metrics | T_ATOM_OVERVIEW_DATA | CREATE_TIME | 371 | 7 | +| devops_metrics | T_PROJECT_USER_DAILY | THE_DATE | 90 | 15 | +| devops_metrics | T_PROJECT_BUILD_SUMMARY_DAILY | THE_DATE | 90 | 15 | + + + +#### 三、非分区表数据清理 + +非分区表采用misc服务的定时任务依据时间、构建数量等条件进行清理，具体配置如下： + +``` +build: + data: + clear: + switch: true # 是否开启自动清理构建历史数据；true:开启，false:不开启 + maxEveryProjectHandleNum: 5 #并发清理项目数量 + monthRange: -1 # 清理多少个月前的普通渠道（BS）流水线运行时数据，对于卡在 stage 审核的流程在清理后就无法继续审核。 + maxKeepNum: 10000 # 普通渠道（BS）流水线最大保留多少条构建历史记录 + codeccDayRange: -14 # 清理多少天前的codecc渠道流水线运行时数据 + codeccMaxKeepNum: 14 # codecc渠道流水线最大保留多少条构建历史记录 + otherMonthRange: -1 # 清理多少天前的其它渠道流水线运行时数据 + otherMaxKeepNum: 500 # 其它渠道流水线最大保留多少条构建历史记录 + clearChannelCodes: "BS,PREBUILD,CI,CODECC" # 支持清理构建上数据的渠道类型 + maxThreadHandleProjectNum: 5 # 开启清理线程最大数量 +``` + diff --git a/docs/overview/db/devops_ci_dispatch.md b/docs/overview/db/devops_ci_dispatch.md index 7169efe9ce8..1f54524f78c 100644 --- a/docs/overview/db/devops_ci_dispatch.md +++ b/docs/overview/db/devops_ci_dispatch.md @@ -438,6 +438,7 @@ | 18 | CONTAINER_HASH_ID | varchar | 128 | 0 | Y | N | | 容器 ID 日志使用 | | 19 | ENV_ID | bigint | 20 | 0 | Y | N | | 第三方构建所属环境 | | 20 | IGNORE_ENV_AGENT_IDS | json | 1073741824 | 0 | Y | N | | 这次调度被排除的 agent 节点 | +| 21 | JOB_ID | varchar | 128 | 0 | Y | N | | 当前构建所属 jobid | **表名：** T_DISPATCH_THIRDPARTY_AGENT_DOCKER_DEBUG diff --git a/docs/overview/db/devops_ci_environment.md b/docs/overview/db/devops_ci_environment.md index 67c04cdedcf..f6f82037fee 100644 --- a/docs/overview/db/devops_ci_environment.md +++ b/docs/overview/db/devops_ci_environment.md @@ -190,6 +190,7 @@ | 1 | ENV_ID | bigint | 20 | 0 | N | Y | | 环境 ID | | 2 | NODE_ID | bigint | 20 | 0 | N | Y | | 节点 ID | | 3 | PROJECT_ID | varchar | 64 | 0 | N | N | | 项目 ID | +| 4 | ENABLE_NODE | bit | 1 | 0 | N | N | b'1' | 是否启用节点 | **表名：** T_ENV_SHARE_PROJECT diff --git a/docs/overview/db/devops_ci_log.md b/docs/overview/db/devops_ci_log.md index 2dd08054973..2cde1010584 100644 --- a/docs/overview/db/devops_ci_log.md +++ b/docs/overview/db/devops_ci_log.md @@ -42,10 +42,12 @@ | 3 | TAG | varchar | 64 | 0 | Y | N | | 标签 | | 4 | SUB_TAG | varchar | 256 | 0 | Y | N | | 子标签 | | 5 | JOB_ID | varchar | 64 | 0 | Y | N | | JOBID | -| 6 | MODE | varchar | 32 | 0 | Y | N | | LogStorageMode | -| 7 | EXECUTE_COUNT | int | 10 | 0 | N | N | | 执行次数 | -| 8 | FINISHED | bit | 1 | 0 | N | N | b'0' | buildisfinishedornot | -| 9 | CREATE_TIME | datetime | 23 | 0 | N | N | CURRENT_TIMESTAMP(3) | 创建时间 | +| 6 | USER_JOB_ID | varchar | 128 | 0 | Y | N | | 真正的 jobId，已经存在的 JOB_ID 字段其实是 containerhashid | +| 7 | STEP_ID | varchar | 64 | 0 | Y | N | | 用户填写的插件 id | +| 8 | MODE | varchar | 32 | 0 | Y | N | | LogStorageMode | +| 9 | EXECUTE_COUNT | int | 10 | 0 | N | N | | 执行次数 | +| 10 | FINISHED | bit | 1 | 0 | N | N | b'0' | buildisfinishedornot | +| 11 | CREATE_TIME | datetime | 23 | 0 | N | N | CURRENT_TIMESTAMP(3) | 创建时间 | **表名：** T_LOG_SUBTAGS diff --git a/docs/overview/db/devops_ci_process.md b/docs/overview/db/devops_ci_process.md index 4b7c7af266c..82aba3c8d41 100644 --- a/docs/overview/db/devops_ci_process.md +++ b/docs/overview/db/devops_ci_process.md @@ -211,7 +211,7 @@ | 4 | BUILD_NUM | int | 10 | 0 | Y | N | 0 | 构建次数 | | 5 | PROJECT_ID | varchar | 64 | 0 | N | N | | 项目 ID | | 6 | PIPELINE_ID | varchar | 34 | 0 | N | N | | 流水线 ID | -| 7 | VERSION | int | 10 | 0 | Y | N | | 版本号 | +| 7 | VERSION | int | 10 | 0 | Y | N | | 编排版本号 | | 8 | START_USER | varchar | 64 | 0 | Y | N | | 启动者 | | 9 | TRIGGER | varchar | 32 | 0 | N | N | | 触发器 | | 10 | START_TIME | timestamp | 19 | 0 | Y | N | | 开始时间 | @@ -241,7 +241,7 @@ | 34 | BUILD_NUM_ALIAS | varchar | 256 | 0 | Y | N | | 自定义构建号 | | 35 | CONCURRENCY_GROUP | varchar | 255 | 0 | Y | N | | 并发时,设定的 group | | 36 | UPDATE_TIME | datetime | 19 | 0 | Y | N | CURRENT_TIMESTAMP | 更新时间 | -| 37 | VERSION_NAME | varchar | 64 | 0 | Y | N | | 版本名称 | +| 37 | VERSION_NAME | varchar | 64 | 0 | Y | N | | 正式版本名称 | | 38 | YAML_VERSION | varchar | 34 | 0 | Y | N | | YAML 的版本标记 | **表名：** T_PIPELINE_BUILD_HISTORY_DEBUG @@ -258,7 +258,7 @@ | 4 | BUILD_NUM | int | 10 | 0 | Y | N | 0 | 构建次数 | | 5 | PROJECT_ID | varchar | 64 | 0 | N | N | | 项目 ID | | 6 | PIPELINE_ID | varchar | 34 | 0 | N | N | | 流水线 ID | -| 7 | VERSION | int | 10 | 0 | Y | N | | 版本号 | +| 7 | VERSION | int | 10 | 0 | Y | N | | 编排版本号 | | 8 | START_USER | varchar | 64 | 0 | Y | N | | 启动者 | | 9 | TRIGGER | varchar | 32 | 0 | N | N | | 触发器 | | 10 | START_TIME | timestamp | 19 | 0 | Y | N | | 开始时间 | @@ -1265,6 +1265,7 @@ | 9 | SECRET_TOKEN | text | 65535 | 0 | Y | N | | Sendtoyourwithhttpheader:X-DEVOPS-WEBHOOK-TOKEN | | 10 | ENABLE | bit | 1 | 0 | N | N | b'1' | 启用 | | 11 | FAILURE_TIME | datetime | 19 | 0 | Y | N | | 失败时间 | +| 12 | SECRET_PARAM | text | 65535 | 0 | Y | N | | 鉴权参数 | **表名：** T_PROJECT_PIPELINE_CALLBACK_HISTORY @@ -1399,3 +1400,4 @@ | 12 | BUILD_NO | text | 65535 | 0 | Y | N | | 构建号 | | 13 | PARAM | mediumtext | 16777215 | 0 | Y | N | | 参数 | | 14 | DELETED | bit | 1 | 0 | Y | N | b'0' | 流水线已被软删除 | +| 15 | INSTANCE_ERROR_INFO | text | 65535 | 0 | Y | N | | 实例化错误信息 | diff --git a/docs/overview/db/devops_ci_store.md b/docs/overview/db/devops_ci_store.md index d6a308d9fde..d4065da48ff 100644 --- a/docs/overview/db/devops_ci_store.md +++ b/docs/overview/db/devops_ci_store.md @@ -40,8 +40,15 @@ | T_REASON | 原因定义表 | | T_REASON_REL | 原因和组件关联关系 | | T_STORE_APPROVE | 审核表 | +| T_STORE_BASE | 研发商店组件基本信息表 | +| T_STORE_BASE_ENV | 研发商店组件执行环境信息表 | +| T_STORE_BASE_ENV_EXT | 研发商店组件执行环境信息扩展表 | +| T_STORE_BASE_EXT | 研发商店组件基本信息扩展表 | +| T_STORE_BASE_FEATURE | 研发商店组件特性信息表 | +| T_STORE_BASE_FEATURE_EXT | 研发商店组件特性信息扩展表 | | T_STORE_BUILD_APP_REL | store 构建与编译环境关联关系表 | | T_STORE_BUILD_INFO | store 组件构建信息表 | +| T_STORE_CATEGORY_REL | 研发商店组件与范畴关联关系表 | | T_STORE_COMMENT | store 组件评论信息表 | | T_STORE_COMMENT_PRAISE | store 组件评论点赞信息表 | | T_STORE_COMMENT_REPLY | store 组件评论回复信息表 | @@ -57,6 +64,7 @@ | T_STORE_INDEX_ELEMENT_DETAIL | 研发商店组件指标要素详情表 | | T_STORE_INDEX_LEVEL_INFO | 研发商店指标等级信息表 | | T_STORE_INDEX_RESULT | 研发商店组件指标结果表 | +| T_STORE_LABEL_REL | 研发商店组件与标签关联关系表 | | T_STORE_MEDIA_INFO | 媒体信息表 | | T_STORE_MEMBER | store 组件成员信息表 | | T_STORE_OPT_LOG | store 操作日志表 | @@ -72,6 +80,7 @@ | T_STORE_STATISTICS | store 统计信息表 | | T_STORE_STATISTICS_DAILY | store 每日统计信息表 | | T_STORE_STATISTICS_TOTAL | store 全量统计信息表 | +| T_STORE_VERSION_LOG | 研发商店组件版本日志表 | | T_TEMPLATE | 模板信息表 | | T_TEMPLATE_CATEGORY_REL | 模板与范畴关联关系表 | | T_TEMPLATE_LABEL_REL | 模板与标签关联关系表 | @@ -174,7 +183,7 @@ | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | | 1 | ID | varchar | 32 | 0 | N | Y | | 主键 ID | | 2 | ATOM_CODE | varchar | 64 | 0 | N | N | | 插件的唯一标识 | -| 3 | TEST_PROJECT_CODE | varchar | 32 | 0 | N | N | | 调试项目编码 | +| 3 | TEST_PROJECT_CODE | varchar | 64 | 0 | N | N | | 调试项目编码 | | 4 | APPROVE_ID | varchar | 32 | 0 | N | N | | 审批 ID | | 5 | CREATOR | varchar | 50 | 0 | N | N | system | 创建者 | | 6 | MODIFIER | varchar | 50 | 0 | N | N | system | 最近修改人 | @@ -703,6 +712,138 @@ | 13 | UPDATE_TIME | datetime | 19 | 0 | N | N | CURRENT_TIMESTAMP | 更新时间 | | 14 | TOKEN | varchar | 64 | 0 | Y | N | | | +**表名：** T_STORE_BASE + +**说明：** 研发商店组件基本信息表 + +**数据列：** + +| 序号 | 名称 | 数据类型 | 长度 | 小数位 | 允许空值 | 主键 | 默认值 | 说明 | +| :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | +| 1 | ID | varchar | 32 | 0 | N | Y | | 主键 ID | +| 2 | STORE_CODE | varchar | 64 | 0 | N | N | | 组件唯一标识 | +| 3 | STORE_TYPE | tinyint | 4 | 0 | N | N | 0 | store 组件类型 | +| 4 | NAME | varchar | 64 | 0 | N | N | | 组件名称 | +| 5 | VERSION | varchar | 256 | 0 | N | N | | 版本号 | +| 6 | STATUS | varchar | 64 | 0 | N | N | | 状态 | +| 7 | STATUS_MSG | varchar | 1024 | 0 | Y | N | | 状态对应的描述，如上架失败原因 | +| 8 | DOCS_LINK | varchar | 256 | 0 | Y | N | | 说明文档链接 | +| 9 | LOGO_URL | varchar | 256 | 0 | Y | N | | logo 地址 | +| 10 | SUMMARY | varchar | 256 | 0 | Y | N | | 组件简介 | +| 11 | DESCRIPTION | mediumtext | 16777215 | 0 | Y | N | | 组件描述 | +| 12 | LATEST_FLAG | bit | 1 | 0 | N | N | | 是否为最新版本，TRUE：最新 FALSE：非最新 | +| 13 | PUBLISHER | varchar | 1024 | 0 | N | N | system | 发布者,对应 T_STORE_PUBLISHER_INFO 表的 PUBLISHER_NAME 字段 | +| 14 | PUB_TIME | datetime | 23 | 0 | Y | N | | 发布时间 | +| 15 | CLASSIFY_ID | varchar | 32 | 0 | N | N | | 所属分类 ID | +| 16 | CREATOR | varchar | 50 | 0 | N | N | system | 创建人 | +| 17 | MODIFIER | varchar | 50 | 0 | N | N | system | 最近修改人 | +| 18 | UPDATE_TIME | datetime | 23 | 0 | N | N | CURRENT_TIMESTAMP(3) | 修改时间 | +| 19 | CREATE_TIME | datetime | 23 | 0 | N | N | CURRENT_TIMESTAMP(3) | 创建时间 | + +**表名：** T_STORE_BASE_ENV + +**说明：** 研发商店组件执行环境信息表 + +**数据列：** + +| 序号 | 名称 | 数据类型 | 长度 | 小数位 | 允许空值 | 主键 | 默认值 | 说明 | +| :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | +| 1 | ID | varchar | 32 | 0 | N | Y | | 主键 | +| 2 | STORE_ID | varchar | 32 | 0 | N | N | | 组件基本信息表 ID | +| 3 | LANGUAGE | varchar | 64 | 0 | Y | N | | 开发语言 | +| 4 | MIN_VERSION | varchar | 20 | 0 | Y | N | | 支持开发语言的最低版本 | +| 5 | PKG_NAME | varchar | 256 | 0 | Y | N | | 包名称 | +| 6 | PKG_PATH | varchar | 1024 | 0 | Y | N | | 包路径 | +| 7 | TARGET | varchar | 256 | 0 | Y | N | | 执行入口 | +| 8 | SHA_CONTENT | varchar | 1024 | 0 | Y | N | | SHA 签名串 | +| 9 | PRE_CMD | text | 65535 | 0 | Y | N | | 执行前置命令 | +| 10 | OS_NAME | varchar | 128 | 0 | Y | N | | 支持的操作系统名称 | +| 11 | OS_ARCH | varchar | 128 | 0 | Y | N | | 支持的操作系统架构 | +| 12 | RUNTIME_VERSION | varchar | 128 | 0 | Y | N | | 运行时版本 | +| 13 | DEFAULT_FLAG | bit | 1 | 0 | N | N | b'1' | 是否为默认环境信息 | +| 14 | CREATOR | varchar | 50 | 0 | N | N | system | 创建人 | +| 15 | MODIFIER | varchar | 50 | 0 | N | N | system | 最近修改人 | +| 16 | UPDATE_TIME | datetime | 23 | 0 | N | N | CURRENT_TIMESTAMP(3) | 修改时间 | +| 17 | CREATE_TIME | datetime | 23 | 0 | N | N | CURRENT_TIMESTAMP(3) | 创建时间 | + +**表名：** T_STORE_BASE_ENV_EXT + +**说明：** 研发商店组件执行环境信息扩展表 + +**数据列：** + +| 序号 | 名称 | 数据类型 | 长度 | 小数位 | 允许空值 | 主键 | 默认值 | 说明 | +| :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | +| 1 | ID | varchar | 32 | 0 | N | Y | | 主键 ID | +| 2 | ENV_ID | varchar | 32 | 0 | N | N | | 组件执行环境信息表 ID | +| 3 | STORE_ID | varchar | 32 | 0 | N | N | | 组件基本信息表 ID | +| 4 | FIELD_NAME | varchar | 64 | 0 | N | N | | 字段名称 | +| 5 | FIELD_VALUE | mediumtext | 16777215 | 0 | N | N | | 字段值 | +| 6 | CREATOR | varchar | 50 | 0 | N | N | system | 创建人 | +| 7 | MODIFIER | varchar | 50 | 0 | N | N | system | 最近修改人 | +| 8 | UPDATE_TIME | datetime | 23 | 0 | N | N | CURRENT_TIMESTAMP(3) | 修改时间 | +| 9 | CREATE_TIME | datetime | 23 | 0 | N | N | CURRENT_TIMESTAMP(3) | 创建时间 | + +**表名：** T_STORE_BASE_EXT + +**说明：** 研发商店组件基本信息扩展表 + +**数据列：** + +| 序号 | 名称 | 数据类型 | 长度 | 小数位 | 允许空值 | 主键 | 默认值 | 说明 | +| :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | +| 1 | ID | varchar | 32 | 0 | N | Y | | 主键 ID | +| 2 | STORE_ID | varchar | 32 | 0 | N | N | | 组件基本信息表 ID | +| 3 | STORE_CODE | varchar | 64 | 0 | N | N | | 组件唯一标识 | +| 4 | STORE_TYPE | tinyint | 4 | 0 | N | N | 0 | store 组件类型 | +| 5 | FIELD_NAME | varchar | 64 | 0 | N | N | | 字段名称 | +| 6 | FIELD_VALUE | mediumtext | 16777215 | 0 | N | N | | 字段值 | +| 7 | CREATOR | varchar | 50 | 0 | N | N | system | 创建人 | +| 8 | MODIFIER | varchar | 50 | 0 | N | N | system | 最近修改人 | +| 9 | UPDATE_TIME | datetime | 23 | 0 | N | N | CURRENT_TIMESTAMP(3) | 修改时间 | +| 10 | CREATE_TIME | datetime | 23 | 0 | N | N | CURRENT_TIMESTAMP(3) | 创建时间 | + +**表名：** T_STORE_BASE_FEATURE + +**说明：** 研发商店组件特性信息表 + +**数据列：** + +| 序号 | 名称 | 数据类型 | 长度 | 小数位 | 允许空值 | 主键 | 默认值 | 说明 | +| :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | +| 1 | ID | varchar | 32 | 0 | N | Y | | 主键 ID | +| 2 | STORE_CODE | varchar | 64 | 0 | N | N | | 组件唯一标识 | +| 3 | STORE_TYPE | tinyint | 4 | 0 | N | N | 0 | store 组件类型 | +| 4 | PUBLIC_FLAG | bit | 1 | 0 | Y | N | b'0' | 是否为公共组件，TRUE：是 FALSE：不是 | +| 5 | RECOMMEND_FLAG | bit | 1 | 0 | Y | N | b'1' | 是否推荐，TRUE：是 FALSE：不是 | +| 6 | CERTIFICATION_FLAG | bit | 1 | 0 | Y | N | b'0' | 是否官方认证，TRUE：是 FALSE：不是 | +| 7 | TYPE | varchar | 32 | 0 | Y | N | | 类型 | +| 8 | RD_TYPE | varchar | 32 | 0 | Y | N | | 研发类型 | +| 9 | WEIGHT | int | 10 | 0 | Y | N | | 权重（数值越大代表权重越高） | +| 10 | CREATOR | varchar | 50 | 0 | N | N | system | 创建人 | +| 11 | MODIFIER | varchar | 50 | 0 | N | N | system | 最近修改人 | +| 12 | UPDATE_TIME | datetime | 23 | 0 | N | N | CURRENT_TIMESTAMP(3) | 修改时间 | +| 13 | CREATE_TIME | datetime | 23 | 0 | N | N | CURRENT_TIMESTAMP(3) | 创建时间 | + +**表名：** T_STORE_BASE_FEATURE_EXT + +**说明：** 研发商店组件特性信息扩展表 + +**数据列：** + +| 序号 | 名称 | 数据类型 | 长度 | 小数位 | 允许空值 | 主键 | 默认值 | 说明 | +| :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | +| 1 | ID | varchar | 32 | 0 | N | Y | | 主键 ID | +| 2 | FEATURE_ID | varchar | 32 | 0 | N | N | | 组件特性信息表 ID | +| 3 | STORE_CODE | varchar | 64 | 0 | N | N | | 组件唯一标识 | +| 4 | STORE_TYPE | tinyint | 4 | 0 | N | N | 0 | store 组件类型 | +| 5 | FIELD_NAME | varchar | 64 | 0 | N | N | | 字段名称 | +| 6 | FIELD_VALUE | text | 65535 | 0 | N | N | | 字段值 | +| 7 | CREATOR | varchar | 50 | 0 | N | N | system | 创建人 | +| 8 | MODIFIER | varchar | 50 | 0 | N | N | system | 最近修改人 | +| 9 | UPDATE_TIME | datetime | 23 | 0 | N | N | CURRENT_TIMESTAMP(3) | 修改时间 | +| 10 | CREATE_TIME | datetime | 23 | 0 | N | N | CURRENT_TIMESTAMP(3) | 创建时间 | + **表名：** T_STORE_BUILD_APP_REL **说明：** store 构建与编译环境关联关系表 @@ -739,6 +880,22 @@ | 10 | SAMPLE_PROJECT_PATH | varchar | 500 | 0 | N | N | | 样例工程路径 | | 11 | STORE_TYPE | tinyint | 4 | 0 | N | N | 0 | store 组件类型 0：插件 1：模板 2：镜像 3：IDE 插件 4：微扩展 | +**表名：** T_STORE_CATEGORY_REL + +**说明：** 研发商店组件与范畴关联关系表 + +**数据列：** + +| 序号 | 名称 | 数据类型 | 长度 | 小数位 | 允许空值 | 主键 | 默认值 | 说明 | +| :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | +| 1 | ID | varchar | 32 | 0 | N | Y | | 主键 | +| 2 | CATEGORY_ID | varchar | 32 | 0 | N | N | | 范畴 ID | +| 3 | STORE_ID | varchar | 32 | 0 | N | N | | 组件 ID | +| 4 | CREATOR | varchar | 50 | 0 | N | N | system | 创建人 | +| 5 | MODIFIER | varchar | 50 | 0 | N | N | system | 最近修改人 | +| 6 | UPDATE_TIME | datetime | 23 | 0 | N | N | CURRENT_TIMESTAMP(3) | 修改时间 | +| 7 | CREATE_TIME | datetime | 23 | 0 | N | N | CURRENT_TIMESTAMP(3) | 创建时间 | + **表名：** T_STORE_COMMENT **说明：** store 组件评论信息表 @@ -1032,6 +1189,22 @@ | 10 | UPDATE_TIME | datetime | 23 | 0 | N | N | CURRENT_TIMESTAMP(3) | 修改时间 | | 11 | CREATE_TIME | datetime | 23 | 0 | N | N | CURRENT_TIMESTAMP(3) | 创建时间 | +**表名：** T_STORE_LABEL_REL + +**说明：** 研发商店组件与标签关联关系表 + +**数据列：** + +| 序号 | 名称 | 数据类型 | 长度 | 小数位 | 允许空值 | 主键 | 默认值 | 说明 | +| :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | +| 1 | ID | varchar | 32 | 0 | N | Y | | 主键 | +| 2 | LABEL_ID | varchar | 32 | 0 | N | N | | 标签 ID | +| 3 | STORE_ID | varchar | 32 | 0 | N | N | | 组件 ID | +| 4 | CREATOR | varchar | 50 | 0 | N | N | system | 创建人 | +| 5 | MODIFIER | varchar | 50 | 0 | N | N | system | 最近修改人 | +| 6 | UPDATE_TIME | datetime | 23 | 0 | N | N | CURRENT_TIMESTAMP(3) | 修改时间 | +| 7 | CREATE_TIME | datetime | 23 | 0 | N | N | CURRENT_TIMESTAMP(3) | 创建时间 | + **表名：** T_STORE_MEDIA_INFO **说明：** 媒体信息表 @@ -1120,6 +1293,7 @@ | 7 | UPDATE_TIME | datetime | 19 | 0 | N | N | CURRENT_TIMESTAMP | 更新时间 | | 8 | STORE_TYPE | tinyint | 4 | 0 | N | N | 0 | 商店组件类型 0：插件 1：模板 2：镜像 3：IDE 插件 | | 9 | BUS_TYPE | varchar | 32 | 0 | N | N | BUILD | 业务类型 BUILD:构建 INDEX:研发商店指标 | +| 10 | PROJECT_CODE | varchar | 64 | 0 | N | N | | 项目代码 | **表名：** T_STORE_PKG_RUN_ENV_INFO @@ -1153,13 +1327,15 @@ | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | | 1 | ID | varchar | 32 | 0 | N | Y | | 主键 ID | | 2 | STORE_CODE | varchar | 64 | 0 | N | N | | store 组件编码 | -| 3 | PROJECT_CODE | varchar | 32 | 0 | N | N | | 用户组所属项目 | +| 3 | PROJECT_CODE | varchar | 64 | 0 | N | N | | 所属项目 | | 4 | TYPE | tinyint | 4 | 0 | N | N | | 类型 | -| 5 | CREATOR | varchar | 50 | 0 | N | N | system | 创建者 | -| 6 | MODIFIER | varchar | 50 | 0 | N | N | system | 修改者 | -| 7 | CREATE_TIME | datetime | 19 | 0 | N | N | CURRENT_TIMESTAMP | 创建时间 | -| 8 | UPDATE_TIME | datetime | 19 | 0 | N | N | CURRENT_TIMESTAMP | 更新时间 | -| 9 | STORE_TYPE | tinyint | 4 | 0 | N | N | 0 | store 组件类型 | +| 5 | VERSION | varchar | 256 | 0 | Y | N | | 版本号 | +| 6 | INSTANCE_ID | varchar | 256 | 0 | Y | N | | 实例 ID | +| 7 | CREATOR | varchar | 50 | 0 | N | N | system | 创建者 | +| 8 | MODIFIER | varchar | 50 | 0 | N | N | system | 修改者 | +| 9 | CREATE_TIME | datetime | 19 | 0 | N | N | CURRENT_TIMESTAMP | 创建时间 | +| 10 | UPDATE_TIME | datetime | 19 | 0 | N | N | CURRENT_TIMESTAMP | 更新时间 | +| 11 | STORE_TYPE | tinyint | 4 | 0 | N | N | 0 | store 组件类型 | **表名：** T_STORE_PUBLISHER_INFO @@ -1306,9 +1482,10 @@ | 6 | DAILY_SUCCESS_NUM | int | 10 | 0 | Y | N | 0 | 每日执行成功数 | | 7 | DAILY_FAIL_NUM | int | 10 | 0 | Y | N | 0 | 每日执行失败总数 | | 8 | DAILY_FAIL_DETAIL | varchar | 4096 | 0 | Y | N | | 每日执行失败详情 | -| 9 | STATISTICS_TIME | datetime | 23 | 0 | N | N | | 统计时间 | -| 10 | CREATE_TIME | datetime | 23 | 0 | N | N | CURRENT_TIMESTAMP(3) | 创建时间 | -| 11 | UPDATE_TIME | datetime | 23 | 0 | N | N | CURRENT_TIMESTAMP(3) | 更新时间 | +| 9 | DAILY_ACTIVE_DURATION | decimal | 16 | 2 | Y | N | | 每日活跃时长，单位：小时 | +| 10 | STATISTICS_TIME | datetime | 23 | 0 | N | N | | 统计时间 | +| 11 | CREATE_TIME | datetime | 23 | 0 | N | N | CURRENT_TIMESTAMP(3) | 创建时间 | +| 12 | UPDATE_TIME | datetime | 23 | 0 | N | N | CURRENT_TIMESTAMP(3) | 更新时间 | **表名：** T_STORE_STATISTICS_TOTAL @@ -1327,9 +1504,27 @@ | 7 | SCORE_AVERAGE | decimal | 4 | 1 | Y | N | 0.0 | 评论均分 | | 8 | PIPELINE_NUM | int | 10 | 0 | Y | N | 0 | 流水线数量 | | 9 | RECENT_EXECUTE_NUM | int | 10 | 0 | Y | N | 0 | 最近执行次数 | -| 10 | CREATE_TIME | datetime | 19 | 0 | N | N | CURRENT_TIMESTAMP | 创建时间 | -| 11 | UPDATE_TIME | datetime | 19 | 0 | N | N | CURRENT_TIMESTAMP | 更新时间 | -| 12 | HOT_FLAG | bit | 1 | 0 | Y | N | b'0' | 是否为受欢迎组件 | +| 10 | RECENT_ACTIVE_DURATION | decimal | 16 | 2 | Y | N | | 总活跃时长，单位：小时 | +| 11 | CREATE_TIME | datetime | 19 | 0 | N | N | CURRENT_TIMESTAMP | 创建时间 | +| 12 | UPDATE_TIME | datetime | 19 | 0 | N | N | CURRENT_TIMESTAMP | 更新时间 | +| 13 | HOT_FLAG | bit | 1 | 0 | Y | N | b'0' | 是否为受欢迎组件 | + +**表名：** T_STORE_VERSION_LOG + +**说明：** 研发商店组件版本日志表 + +**数据列：** + +| 序号 | 名称 | 数据类型 | 长度 | 小数位 | 允许空值 | 主键 | 默认值 | 说明 | +| :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | +| 1 | ID | varchar | 32 | 0 | N | Y | | 主键 | +| 2 | STORE_ID | varchar | 32 | 0 | N | N | | 组件 ID | +| 3 | RELEASE_TYPE | tinyint | 4 | 0 | N | N | | 发布类型，0：新上架 1：非兼容性升级 2：兼容性功能更新 3：兼容性问题修正 | +| 4 | CONTENT | text | 65535 | 0 | N | N | | 版本日志内容 | +| 5 | CREATOR | varchar | 50 | 0 | N | N | system | 创建人 | +| 6 | MODIFIER | varchar | 50 | 0 | N | N | system | 最近修改人 | +| 7 | UPDATE_TIME | datetime | 23 | 0 | N | N | CURRENT_TIMESTAMP(3) | 修改时间 | +| 8 | CREATE_TIME | datetime | 23 | 0 | N | N | CURRENT_TIMESTAMP(3) | 创建时间 | **表名：** T_TEMPLATE diff --git a/docs/overview/db/sharding_dev.md b/docs/overview/db/sharding_dev.md new file mode 100644 index 00000000000..bdaff4d2c83 --- /dev/null +++ b/docs/overview/db/sharding_dev.md @@ -0,0 +1,276 @@ +# 蓝盾分库分表开发指南 +## 背景 + + 随着蓝盾的构建量快速增长， 快速增长的构建量带来的请求量和数据给蓝盾数据带来了巨大的压力 ，巨大的QPS量和数据量已经让单机数据库达到了瓶颈，蓝盾db支持分库分表势在必行。 + + + +## 开发步骤 + +#### 一、确定分库分表数量 + +​ 根据流水线构建趋势图确定分库分表数量，保证分库分表数量合理。 + + + +#### 二、增加分库分表配置 + +##### 1、在devops_ci_project数据库的T_DATA_SOURCE表插入分区库的记录，T_TABLE_SHARDING_CONFIG表插入分区表的记录（只分库不分表的情况下可以不用配置），数据库表的结构如下： + +``` +CREATE TABLE `T_DATA_SOURCE` ( + `ID` varchar(32) NOT NULL DEFAULT '' COMMENT '主键ID', + `MODULE_CODE` varchar(64) NOT NULL DEFAULT '' COMMENT '模块标识', + `DATA_SOURCE_NAME` varchar(128) NOT NULL DEFAULT '' COMMENT '数据源名称', + `FULL_FLAG` bit(1) DEFAULT b'0' COMMENT '容量是否满标识 true：是，false：否', + `CREATOR` varchar(50) NOT NULL DEFAULT 'system' COMMENT '创建者', + `MODIFIER` varchar(50) NOT NULL DEFAULT 'system' COMMENT '修改者', + `UPDATE_TIME` datetime(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) COMMENT '修改时间', + `CREATE_TIME` datetime(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) COMMENT '创建时间', + `CLUSTER_NAME` varchar(64) NOT NULL DEFAULT '' COMMENT '集群名称', + `DS_URL` varchar(1024) DEFAULT NULL COMMENT '数据源URL地址', + `TAG` varchar(128) DEFAULT NULL COMMENT '数据源标签', + `TYPE` varchar(32) NOT NULL DEFAULT 'DB' COMMENT '数据库类型，DB:普通数据库，ARCHIVE_DB:归档数据库', + PRIMARY KEY (`ID`), + UNIQUE KEY `UNI_INX_TDS_CLUSTER_MODULE_TYPE_NAME` (`CLUSTER_NAME`,`MODULE_CODE`,`TYPE`,`DATA_SOURCE_NAME`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='模块数据源配置'; + +sql示例: +INSERT INTO devops_ci_project.T_DATA_SOURCE +(ID, MODULE_CODE, DATA_SOURCE_NAME, FULL_FLAG, CREATOR, MODIFIER, UPDATE_TIME, CREATE_TIME, CLUSTER_NAME, DS_URL, TAG, `TYPE`) +VALUES('eae3670d3716427881c93fde46e28532', 'PROCESS', 'ds_0', 0, 'system', 'system', '2022-01-11 15:42:20.280', '2022-01-11 15:42:20.280', 'prod', 'xxxxx', NULL, 'DB'); + +INSERT INTO devops_ci_project.T_DATA_SOURCE +(ID, MODULE_CODE, DATA_SOURCE_NAME, FULL_FLAG, CREATOR, MODIFIER, UPDATE_TIME, CREATE_TIME, CLUSTER_NAME, DS_URL, TAG, `TYPE`) +VALUES('eae3670d3716427881c93fde46e28533', 'PROCESS', 'ds_1', 0, 'system', 'system', '2022-01-19 11:04:45.529', '2022-01-11 15:42:20.280', 'prod', 'xxxxx', NULL, 'DB'); + +INSERT INTO devops_ci_project.T_DATA_SOURCE +(ID, MODULE_CODE, DATA_SOURCE_NAME, FULL_FLAG, CREATOR, MODIFIER, UPDATE_TIME, CREATE_TIME, CLUSTER_NAME, DS_URL, TAG, `TYPE`) +VALUES('eae3670d3716427881c93fde46e26537', 'PROCESS', 'archive_ds_0', 0, 'system', 'system', '2024-01-03 18:31:10.459', '2024-01-03 18:31:10.459', 'stream', 'xxxxx', 'archive', 'ARCHIVE_DB'); + +CREATE TABLE `T_TABLE_SHARDING_CONFIG` ( + `ID` varchar(32) NOT NULL DEFAULT '' COMMENT '主键ID', + `CLUSTER_NAME` varchar(64) NOT NULL DEFAULT '' COMMENT '集群名称', + `MODULE_CODE` varchar(64) NOT NULL DEFAULT '' COMMENT '模块标识', + `TABLE_NAME` varchar(128) NOT NULL DEFAULT '' COMMENT '数据库表名称', + `SHARDING_NUM` int(11) NOT NULL DEFAULT '5' COMMENT '分表数量', + `CREATOR` varchar(50) NOT NULL DEFAULT 'system' COMMENT '创建者', + `MODIFIER` varchar(50) NOT NULL DEFAULT 'system' COMMENT '修改者', + `UPDATE_TIME` datetime(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) COMMENT '修改时间', + `CREATE_TIME` datetime(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) COMMENT '创建时间', + PRIMARY KEY (`ID`), + UNIQUE KEY `UNI_INX_TTSC_CLUSTER_MODULE_NAME` (`CLUSTER_NAME`,`MODULE_CODE`,`TABLE_NAME`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='数据库表分片配置'; +``` + + + +##### 2、在蓝盾对应微服务（以process为例）的yml文件增加分库分表数据源配置，模板如下（**db的序号不能随便更改，后续如果新增分区库需要往下按序号追加**）： + +``` +spring: + datasource: + # 普通库数据源配置（勿随便变更配置项的顺序） + dataSourceConfigs: + - index: 0 + url: jdbc:mysql://xxx01:10000/devops_ci_process?useSSL=false&autoReconnect=true&serverTimezone=GMT%2B8&useUnicode=true&characterEncoding=utf8&allowMultiQueries=true + username: ENC(xxxxx) + password: ENC(xxxxx) + - index: 1 + url: jdbc:mysql://xxx02:10000/devops_ci_process?useSSL=false&autoReconnect=true&serverTimezone=GMT%2B8&useUnicode=true&characterEncoding=utf8&allowMultiQueries=true + username: ENC(xxxxx) + password: ENC(xxxxx) + # 归档库数据源配置（勿随便变更配置项的顺序） + archiveDataSourceConfigs: + - index: 0 + url: jdbc:mysql://archivexxx01:10000/devops_ci_archive_process?useSSL=false&autoReconnect=true&serverTimezone=GMT%2B8&useUnicode=true&characterEncoding=utf8&allowMultiQueries=true + username: ENC(xxxxx) + password: ENC(xxxxx) +``` + + + +##### 3、在蓝盾对应微服务（以process为例）的yml文件增加公共分片规则配置 + +``` +# shardingsphere分片规则配置 +sharding: + databaseShardingStrategy: + algorithmClassName: "com.tencent.devops.process.sharding.BkProcessDatabaseShardingAlgorithm" # 普通业务库分库路由算法实现类 + archiveAlgorithmClassName: "com.tencent.devops.process.sharding.BkProcessArchiveDatabaseShardingAlgorithm" # 归档库分库路由算法实现类 + shardingField: PROJECT_ID # 分库分片键 + tableShardingStrategy: + archiveAlgorithmClassName: "com.tencent.devops.process.sharding.BkProcessArchiveTableShardingAlgorithm" # 归档库分表路由算法实现类 + shardingField: PROJECT_ID # 分表分片键 + archiveFlag: Y # 是否使用归档库标识；Y：是，N：否 + defaultFlag: Y # 是否使用默认库标识；Y：是，N：否 + routing: + cacheSize: 100000 # 缓存分片规则最大数量 + migration: + timeout: 2 # 迁移项目超时时间，单位：小时 + maxProjectCount: 5 # 同时迁移项目最大数量 + processDbMicroServices: "process,engine,misc,lambda" #使用process数据库的微服务 + sourceDbDataDeleteFlag: false # 迁移成功后是否删除原库数据 + tableShardingStrategy: + defaultShardingNum: 1 # 默认分表数量 +``` + + + +##### 4、在蓝盾对应微服务（以process为例）的yml文件增表的分片规则配置 + +``` +spring: + profiles: prod + datasource: + tableRuleConfigs: + - index: 0 # 序号 + name: T_AUDIT_RESOURCE # 表名 + databaseShardingStrategy: SHARDING # 分库策略 + tableShardingStrategy: SHARDING # 分表策略（可为空，不分表的情况下可以不配） + shardingNum: 1 # 分表数量（可为空，不分表的情况下可以不配） + - index: 1 # 序号 + name: T_PIPELINE_RULE # 表名 + broadcastFlag: true # 是否为广播表（所有分区库中该表的数据都一样） +``` + + + +##### 5、分布式ID配置 + + 蓝盾数据库表的ID分为UUID(**全局唯一**)和数据库自增长主键ID(**非全局唯一**)，我们为数据库自增长主键ID的表开发了基于号段模式实现的分布式ID方案 ，该方案需在devops_ci_project数据库的T_LEAF_ALLOC表插入业务ID生成方案配置的记录 ，数据库表的结构如下： + +``` +CREATE TABLE `T_LEAF_ALLOC` ( + `BIZ_TAG` varchar(128) NOT NULL DEFAULT '' COMMENT '业务标签', + `MAX_ID` bigint(20) NOT NULL DEFAULT '1' COMMENT '当前最大ID值', + `STEP` int(11) NOT NULL COMMENT '步长,每一次请求获取的ID个数', + `DESCRIPTION` varchar(256) DEFAULT NULL COMMENT '说明', + `UPDATE_TIME` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', + PRIMARY KEY (`BIZ_TAG`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='ID管理数据表'; +``` + + + +##### 6、按照分表数量新建表（如果只分库不分表可以忽略该步骤） + +示例：T_ATOM_OVERVIEW_DATA表分表数量为2，需要新建T_ATOM_OVERVIEW_DATA（**供jooq生成数据模型，不存数据**）、T_ATOM_OVERVIEW_DATA_0、T_ATOM_OVERVIEW_DATA_1 这三张表。 + + + +#### 三、业务逻辑开发 + +1、涉及分区表的sql需加上分片键，保证sql路由到目标db或者表执行。 + +``` +xxxxxx WHERE PROJECT_ID = "xxx" +``` + + + +2、如果业务要用数据库自增长主键ID需要 调取project服务封装的分布式ID生成接口获取ID,调用代码模板如下： + +``` +client.get(ServiceAllocIdResource::class).generateSegmentId(TEMPLATE_BIZ_TAG_NAME).data +``` + + + +#### 四：路由分配规则二次开发（非必须） + +在为项目分片路由规则的时候默认采用的是随机分配算法，如果想保证各分区库或者分区表的访问流量、数据量相对比较均衡，可以根据自身业务特点对项目的路由规则进行二次开发（列如可以计算每个分区库或者分区表一段时间的总构建量，新建项目的时候将总构建量相对较小的db或者表分配给该项目） + + /** + * 获取可用数据源名称 + * @param clusterName db集群名称 + * @param moduleCode 模块代码 + * @param ruleType 规则类型 + * @param dataSourceNames 数据源名称集合 + * @return 可用数据源名称 + */ + override fun getValidDataSourceName( + clusterName: String, + moduleCode: SystemModuleEnum, + ruleType: ShardingRuleTypeEnum, + dataSourceNames: List + ): String { + // 从可用的数据源中随机选择一个分配给该项目 + val maxSizeIndex = dataSourceNames.size - 1 + val randomIndex = (0..maxSizeIndex).random() + return dataSourceNames[randomIndex] + } + + /** + * 获取可用数据库表名称 + * @param ruleType 规则类型 + * @param dataSourceName 数据源名称 + * @param tableShardingConfig 分表配置 + * @return 可用数据库表名称 + */ + override fun getValidTableName( + ruleType: ShardingRuleTypeEnum, + dataSourceName: String, + tableShardingConfig: TableShardingConfig + ): String { + // 从可用的数据库表中随机选择一个分配给该项目 + val tableName = tableShardingConfig.tableName + val shardingNum = tableShardingConfig.shardingNum + return if (shardingNum > 1) { + val maxSizeIndex = shardingNum - 1 + val randomIndex = (0..maxSizeIndex).random() + "${tableName}_$randomIndex" + } else { + tableName + } + } +##### + +#### 五：数据迁移（非必须） + +在原来db的数据量比较大的情况下，我们会增加新的db组成新的db集群以实现扩容，如果想把原db的部分数据迁移到db以减轻原db的压力可以使用蓝盾迁移项目数据的接口实现，具体的步骤如下： + +##### 1、为新增的db配置指定tag + +示例： + +INSERT INTO devops_ci_project.T_DATA_SOURCE +(ID, MODULE_CODE, DATA_SOURCE_NAME, FULL_FLAG, CREATOR, MODIFIER, UPDATE_TIME, CREATE_TIME, CLUSTER_NAME, DS_URL, TAG, `TYPE`) +VALUES('eae3670d3716427881c93fde46e28501', 'PROCESS', 'ds_3', 0, 'system', 'system', '2023-08-29 15:44:40.694', '2023-08-29 15:44:40.694', 'prod', 'xxx', '**tagxxx**', 'DB'); + + + +##### 2、在misc服务调用蓝盾迁移项目数据的接口 + +``` +curl -X PUT "http://xxx/api/op/process/db/projects/{projectId}/data/migrate?cancelFlag=true&dataTag=tagxxx" -H "accept: application/json" -H "X-DEVOPS-UID: xxx" +``` + +参数说明： + +| 参数名称 | 参数类型 | 说明 | +| ------------ | -------- | ---------------------- | +| X-DEVOPS-UID | header | 迁移触发人 | +| projectId | path | 迁移项目 | +| cancelFlag | query | 是否取消正在运行的构建 | +| dataTag | query | 迁移库对应的数据标签 | + + + +**注意：** + +1、迁移项目前需要向项目的负责人明确迁移时是否要取消当前正在运行的构建（**不允许取消构建可能存在迁移后数据不一致的情况**），如果允许取消：cancelFlag字段传true，不允许取消：cancelFlag字段传false。 + +2、迁移失败后该项目的流量还是处于锁定状态，如果不继续迁移了，需要调用project微服务的项目流量解锁接口进行解锁，该接口调用示例如下： + +``` +curl -X PUT "http://xxx/api/op/projects/{projectId}/pipeline/build/unlock" -H "accept: application/json" -H "X-DEVOPS-UID: xxx" +``` + + + +##### 3、把新增的db配置的tag置为null + +把新增的db配置的tag置为null后，则说明新增db已经正式加入默认集群，为新项目创建分片路由规则时也可能会将新增的db分配给该项目使用。 + +UPDATE devops_ci_project.T_DATA_SOURCE SET TAG = **null** WHERE ID = 'xxxxxx'; \ No newline at end of file diff --git a/helm-charts/core/ci/Chart.lock b/helm-charts/core/ci/Chart.lock index c769f98630d..16713a585e1 100644 --- a/helm-charts/core/ci/Chart.lock +++ b/helm-charts/core/ci/Chart.lock @@ -25,6 +25,6 @@ dependencies: version: 10.30.6 - name: kubernetes-manager repository: file://./local_chart/kubernetes-management - version: 0.0.37 -digest: sha256:f39b897b438471cfb4cc9e98a3c4c44ec9d8950c1bb7b658dda8a354cad922a3 -generated: "2023-08-09T15:09:12.834702422+08:00" + version: 0.0.45 +digest: sha256:bb11b7ac0e3487504f5563cd2b170d04038fc8971aaecbaca3dc5ecdcb792a43 +generated: "2024-06-21T18:05:57.191350067+08:00" diff --git a/helm-charts/core/ci/base/values.yaml b/helm-charts/core/ci/base/values.yaml index 58cb936e802..0094c6c3794 100644 --- a/helm-charts/core/ci/base/values.yaml +++ b/helm-charts/core/ci/base/values.yaml @@ -174,7 +174,8 @@ elasticsearch: rabbitmq: enabled: true image: - repository: bkci/rabbitmq-with-delay + registry: docker.io/bkci + repository: rabbitmq-with-delay tag: 0.0.1 persistence: size: 10Gi diff --git a/helm-charts/core/ci/build_chart.py b/helm-charts/core/ci/build_chart.py index 1bfdc77e328..ddb24e417cf 100755 --- a/helm-charts/core/ci/build_chart.py +++ b/helm-charts/core/ci/build_chart.py @@ -78,7 +78,7 @@ else: camelize_set.add(camelize_key) if line.replace(key, "").strip().endswith(":"): - line = line.replace(key, '{{ .Values.config.'+camelize_key+' | quote }}') + line = line.replace(key, '{{ .Values.config.'+camelize_key+' | default "" | quote }}') else: line = line.replace(key, '{{ .Values.config.'+camelize_key+' }}') new_file.write(line) @@ -106,7 +106,7 @@ gateway_config_file.write(env+": "+include_dict[camelize_key]+"\n") else: camelize_set.add(camelize_key) - gateway_config_file.write(env+": "+'{{ .Values.config.'+camelize_key+" | quote }}\n") + gateway_config_file.write(env+": "+'{{ .Values.config.'+camelize_key+' | default "" | quote }}\n') # 前端文件 for root, dirs, files in os.walk(frontend_path): for frontend_file in files: @@ -125,7 +125,7 @@ gateway_config_file.write(env+": "+include_dict[camelize_key]+"\n") else: camelize_set.add(camelize_key) - gateway_config_file.write(env+": "+'{{ .Values.config.'+camelize_key+" | quote }}\n") + gateway_config_file.write(env+": "+'{{ .Values.config.'+camelize_key+' | default "" | quote }}\n') gateway_config_file.write('NAMESPACE: {{ .Release.Namespace }}\n') gateway_config_file.write('CHART_NAME: {{ include "bkci.names.fullname" . }}\n') gateway_config_file.write('{{ end }}') diff --git a/helm-charts/core/ci/charts/kubernetes-manager-0.0.45.tgz b/helm-charts/core/ci/charts/kubernetes-manager-0.0.45.tgz index 92f2b7f9809..bf049c9fea9 100644 Binary files a/helm-charts/core/ci/charts/kubernetes-manager-0.0.45.tgz and b/helm-charts/core/ci/charts/kubernetes-manager-0.0.45.tgz differ diff --git a/helm-charts/core/ci/local_chart/kubernetes-management/Chart.yaml b/helm-charts/core/ci/local_chart/kubernetes-management/Chart.yaml index b1de242ebb1..87ba8e44885 100644 --- a/helm-charts/core/ci/local_chart/kubernetes-management/Chart.yaml +++ b/helm-charts/core/ci/local_chart/kubernetes-management/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: kubernetes-manager description: A Helm chart for BlueKing CI Kubernetes Manager type: application -version: 0.0.37 +version: 0.0.45 appVersion: 0.0.31 home: https://github.com/Tencent/bk-ci diff --git a/helm-charts/core/ci/local_chart/kubernetes-management/templates/deployment.yaml b/helm-charts/core/ci/local_chart/kubernetes-management/templates/deployment.yaml index 3038bb8ebd8..26c681cde45 100644 --- a/helm-charts/core/ci/local_chart/kubernetes-management/templates/deployment.yaml +++ b/helm-charts/core/ci/local_chart/kubernetes-management/templates/deployment.yaml @@ -76,6 +76,14 @@ spec: value: {{ .Values.multiCluster.enabled | quote }} - name: DEFAULT_NAMESPACE value: {{ .Values.multiCluster.defaultNamespace }} + {{- if .Values.kubernetesManager.docker.enable }} + - name: DOCKER_HOST + value: tcp://localhost:2375 + {{- end}} + {{- if .Values.kubernetesManager.debug }} + - name: KUBERNETES_MANAGER_DEBUG_ENABLE + value: "true" + {{- end}} workingDir: /data/workspace/kubernetes-manager livenessProbe: tcpSocket: @@ -99,8 +107,22 @@ spec: mountPath: /data/workspace/kubernetes-manager/config readOnly: true {{- end}} - {{- if .Values.configmap.enabled}} + {{- if .Values.kubernetesManager.docker.enable }} + - name: kuberentes-manager-docker + image: {{ .Values.kubernetesManager.docker.image }} + command: ["dockerd", "--host", "tcp://localhost:2375"] + {{- if .Values.kubernetesManager.docker.resources }} + resources: {{- toYaml .Values.kubernetesManager.docker.resources | nindent 12 }} + {{- end }} + securityContext: + privileged: true + volumeMounts: + - name: docker-graph-storage + mountPath: /var/lib/docker + {{- end }} + volumes: + {{- if .Values.configmap.enabled}} - name: kubernetes-manager-config configMap: name: kubernetes-manager @@ -110,6 +132,10 @@ spec: {{- if .Values.kubeConfig.useKubeConfig}} - key: kubeConfig.yaml path: kubeConfig.yaml - {{- end}} + {{- end}} + {{- if .Values.kubernetesManager.docker.enable }} + - name: docker-graph-storage + emptyDir: {} + {{- end}} {{- end}} {{- end -}} diff --git a/helm-charts/core/ci/local_chart/kubernetes-management/templates/kubernetes-manager-configmap.yaml b/helm-charts/core/ci/local_chart/kubernetes-management/templates/kubernetes-manager-configmap.yaml index a8dd052b646..c2a930b027a 100644 --- a/helm-charts/core/ci/local_chart/kubernetes-management/templates/kubernetes-manager-configmap.yaml +++ b/helm-charts/core/ci/local_chart/kubernetes-management/templates/kubernetes-manager-configmap.yaml @@ -135,6 +135,9 @@ data: rsaPrivateKey: | {{- .Values.kubernetesManager.apiserver.auth.rsaPrivateKey | nindent 10 }} + docker: + enable: {{ .Values.kubernetesManager.docker.enable }} + {{ if .Values.kubeConfig.useKubeConfig -}} kubeConfig.yaml: | {{- .Values.kubeConfig.content | nindent 4 }} diff --git a/helm-charts/core/ci/local_chart/kubernetes-management/values.yaml b/helm-charts/core/ci/local_chart/kubernetes-management/values.yaml index c11f424f3c2..dee96681d3c 100644 --- a/helm-charts/core/ci/local_chart/kubernetes-management/values.yaml +++ b/helm-charts/core/ci/local_chart/kubernetes-management/values.yaml @@ -94,6 +94,7 @@ service: # kubernetesManager Deployment kubernetesManager: enabled: true + debug: false replicas: 1 resources: requests: @@ -147,11 +148,23 @@ kubernetesManager: apiToken: key: Devops-Token value: landun - rsaPrivateKey: | + rsaPrivateKey: "" volumeMount: # 流水线构建工作空间和agent日志在容器内的挂载点 dataPath: /data/devops/workspace logPath: /data/devops/logs + # manager使用docker相关配置，会启用特权模式容器 + docker: + enable: true + image: docker:24.0.1-dind + resources: + requests: + cpu: 50m + memory: 512Mi + limits: + cpu: 100m + memory: 1024Mi + dockerInit: # 是否使用当前chart的 dockerinit.sh useDockerInit: true diff --git a/helm-charts/core/ci/templates/gateway/deployment.yaml b/helm-charts/core/ci/templates/gateway/deployment.yaml index f1617fa095b..b5329e0cf51 100644 --- a/helm-charts/core/ci/templates/gateway/deployment.yaml +++ b/helm-charts/core/ci/templates/gateway/deployment.yaml @@ -63,8 +63,6 @@ spec: - "-c" - | cp -r /data/workspace/frontend/* /tmp/frontend/ - sysctl -w net.ipv4.tcp_tw_reuse=0 - sysctl -w net.ipv4.tcp_max_tw_buckets=16384 containers: - name: gateway image: {{ include "bkci-gateway.image" . }} diff --git a/helm-charts/core/ci/templates/init/init.bkrepo.yaml b/helm-charts/core/ci/templates/init/init.bkrepo.yaml index ef3035e3f68..9d16cb9dd4e 100644 --- a/helm-charts/core/ci/templates/init/init.bkrepo.yaml +++ b/helm-charts/core/ci/templates/init/init.bkrepo.yaml @@ -33,7 +33,7 @@ spec: REPO_CREATE_GENERIC_PATH="{{ .Values.config.bkRepoApiUrl }}/repository/api/repo/create" REPO_INIT_GENERIC_METADATA_PATH="{{ .Values.config.bkRepoApiUrl }}/generic/" create_repo_project_name_init_plugintransfer_project_generic (){ - for i in bk-store + for i in bk-store bkcdn do ret=0 echo "CI project is $i -------------------------------------------------->" diff --git a/helm-charts/core/ci/templates/init/init.iam-rbac.yaml b/helm-charts/core/ci/templates/init/init.iam-rbac.yaml index b802482e8a2..dd9d035f73e 100644 --- a/helm-charts/core/ci/templates/init/init.iam-rbac.yaml +++ b/helm-charts/core/ci/templates/init/init.iam-rbac.yaml @@ -21,13 +21,13 @@ spec: - name: init-iam image: {{ include "bkci-backend.image" . }} imagePullPolicy: {{ .Values.backendImage.pullPolicy }} - workingDir: /data/workspace/support-files/ + workingDir: /data/workspace/support-files/bkiam-rbac command: - "/bin/bash" - "-c" - | # 修改auth链接 - sed -i 's/ci-auth.service.consul:21936/{{- .Values.config.bkCiHost -}}\/auth/g' bkiam-rbac/*.json + sed -i 's/ci-auth.service.consul:21936/{{- .Values.config.bkCiHost -}}\/auth/g' *.json # 导入模型 for i in $(find . -name '*.json'|sort) do @@ -47,8 +47,8 @@ spec: # 注册auth回调 echo "{{ include "bkci.names.fullname" . }}-auth is available"; - sed -i 's/bk-ci.service.consul/{{ include "bkci.names.fullname" . }}-gateway.{{ .Release.Namespace }}/g' ms-init/auth/iam-callback-resource-registere.conf - iam_json_file="ms-init/auth/iam-callback-resource-registere.conf" + sed -i 's/bk-ci.service.consul/{{ include "bkci.names.fullname" . }}-gateway.{{ .Release.Namespace }}/g' ../ms-init/auth/iam-callback-resource-registere.conf + iam_json_file="../ms-init/auth/iam-callback-resource-registere.conf" curl -X POST -H "Content-Type:application/json" -d "@$iam_json_file" "http://{{ include "bkci.names.fullname" . }}-auth.{{ .Release.Namespace }}.svc.cluster.local/api/op/auth/iam/callback/" restartPolicy: OnFailure {{- end -}} diff --git a/helm-charts/core/ci/templates/init/init.iam.yaml b/helm-charts/core/ci/templates/init/init.iam.yaml deleted file mode 100644 index 984171b3307..00000000000 --- a/helm-charts/core/ci/templates/init/init.iam.yaml +++ /dev/null @@ -1,56 +0,0 @@ -# 初始化iam -{{ if .Values.init.iam }} -{{- if eq .Values.config.bkCiAuthProvider "bk_login_v3" -}} -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ include "bkci.names.fullname" . }}-init-iam - labels: {{- include "bkci.labels.standard" . | nindent 4 }} - app.kubernetes.io/component: init-iam - annotations: - "helm.sh/hook": post-install,post-upgrade - "helm.sh/hook-weight": "-4" - "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded -spec: - template: - metadata: - labels: {{- include "bkci.labels.standard" . | nindent 8 }} - app.kubernetes.io/component: init-iam - spec: - containers: - - name: init-iam - image: {{ include "bkci-backend.image" . }} - imagePullPolicy: {{ .Values.backendImage.pullPolicy }} - workingDir: /data/workspace/support-files/ - {{ $mysqlData := split ":" (include "bkci.mysqlAddr" .) }} - command: - - "/bin/bash" - - "-c" - - | - # 修改auth链接 - sed -i 's/ci-auth.service.consul:21936/{{- .Values.config.bkCiHost -}}\/auth/g' bkiam/*.json - # 导入模型 - for i in $(find . -name '*.json'|sort) - do - python3 bkiam_do_migrate.py -t {{ .Values.config.bkIamPrivateUrl }} -a "{{ .Values.config.bkCiAppCode }}" -s "{{ .Values.config.bkCiAppToken }}" -f $i - done - - services="auth" - for service in $services - do - until curl --connect-timeout 3 -m 1 -s "http://{{ include "bkci.names.fullname" . }}-$service.{{ .Release.Namespace }}.svc.cluster.local" > nohup - do - echo "waiting for {{ include "bkci.names.fullname" . }}-$service"; - sleep 2; - done - echo "{{ include "bkci.names.fullname" . }}-$service is available"; - done - - # 注册auth回调 - echo "{{ include "bkci.names.fullname" . }}-auth is available"; - sed -i 's/bk-ci.service.consul/{{ include "bkci.names.fullname" . }}-gateway.{{ .Release.Namespace }}/g' ms-init/auth/iam-callback-resource-registere.conf - iam_json_file="ms-init/auth/iam-callback-resource-registere.conf" - curl -X POST -H "Content-Type:application/json" -d "@$iam_json_file" "http://{{ include "bkci.names.fullname" . }}-auth.{{ .Release.Namespace }}.svc.cluster.local/api/op/auth/iam/callback/" - restartPolicy: OnFailure -{{- end -}} -{{- end -}} diff --git a/src/agent/agent-slim/cmd/translation_generator/translation_generator.go b/src/agent/agent-slim/cmd/translation_generator/translation_generator.go index c6bb4612542..f48fc746b07 100644 --- a/src/agent/agent-slim/cmd/translation_generator/translation_generator.go +++ b/src/agent/agent-slim/cmd/translation_generator/translation_generator.go @@ -156,7 +156,7 @@ func main() { } } -// 生成器保存分析的状态。 主要用来缓冲 format.Source 的输出。 +// Generator 生成器保存分析的状态。 主要用来缓冲 format.Source 的输出。 type Generator struct { buf bytes.Buffer // 累计输出 } diff --git a/src/agent/agent-slim/pkg/constant/constant.go b/src/agent/agent-slim/pkg/constant/constant.go index be131a15722..38a656b6600 100644 --- a/src/agent/agent-slim/pkg/constant/constant.go +++ b/src/agent/agent-slim/pkg/constant/constant.go @@ -1,11 +1,11 @@ package constant const ( - // 构建机默认国际化语言 - DEFAULT_LANGUAGE_TYPE = "zh_CN" - // 构建机接取任务间隔时间 + // DefaultLanguageType 构建机默认国际化语言 + DefaultLanguageType = "zh_CN" + // BuildIntervalInSeconds 构建机接取任务间隔时间 BuildIntervalInSeconds = 5 - // api 鉴权的头信息 + // AuthHeaderBuildType api 鉴权的头信息 AuthHeaderBuildType = "X-DEVOPS-BUILD-TYPE" // 构建类型 AuthHeaderProjectId = "X-DEVOPS-PROJECT-ID" // 项目ID AuthHeaderAgentId = "X-DEVOPS-AGENT-ID" // Agent ID diff --git a/src/agent/agent-slim/pkg/i18n/i18n.go b/src/agent/agent-slim/pkg/i18n/i18n.go index a2e22d7960d..9b48a584ed6 100644 --- a/src/agent/agent-slim/pkg/i18n/i18n.go +++ b/src/agent/agent-slim/pkg/i18n/i18n.go @@ -44,7 +44,7 @@ import ( var localizer *localizerType -var defaultLocalTag = language.Make(constant.DEFAULT_LANGUAGE_TYPE) +var defaultLocalTag = language.Make(constant.DefaultLanguageType) type localizerType struct { nowLocalizer language.Tag diff --git a/src/agent/agent-slim/pkg/job/build_manager.go b/src/agent/agent-slim/pkg/job/build_manager.go index bd919005a05..7d078e7c5f7 100644 --- a/src/agent/agent-slim/pkg/job/build_manager.go +++ b/src/agent/agent-slim/pkg/job/build_manager.go @@ -27,7 +27,7 @@ var GBuildManager *buildManager func init() { GBuildManager = &buildManager{ lock: &sync.RWMutex{}, - instances: make(map[int]*buildData, 0), + instances: make(map[int]*buildData), } } diff --git a/src/agent/agent/.gitignore b/src/agent/agent/.gitignore new file mode 100644 index 00000000000..b59f7e3a95a --- /dev/null +++ b/src/agent/agent/.gitignore @@ -0,0 +1 @@ +test/ \ No newline at end of file diff --git a/src/agent/agent/README.md b/src/agent/agent/README.md index 11efd728a30..388c396458b 100644 --- a/src/agent/agent/README.md +++ b/src/agent/agent/README.md @@ -1,6 +1,7 @@ # Agent(Golang) -构建Agent是指包含了agent进程监控和调度部分逻辑的代码，不包含与流水线交互的构建类业务逻辑代码，需要与另外一个worker(kotlin) 一起整合才能是完整的Agent包。 +构建Agent是指包含了agent进程监控和调度部分逻辑的代码，不包含与流水线交互的构建类业务逻辑代码，需要与另外一个worker(kotlin) +一起整合才能是完整的Agent包。 ## Agent二进制程序编译 @@ -14,7 +15,8 @@ windows编译，执行脚本 `build_windows.bat` 执行以上编译脚本或命令后，会在 `bin` 目录下生成对应的可执行文件。 -比如执行 `make clean build_linux` 命令会在 `bin` 目录下生成 `devopsDaemon_linux`、`devopsAgent_linux`、`upgrader_linux` 文件，其他系统依此类推。 +比如执行 `make clean build_linux` 命令会在 `bin` 目录下生成 `devopsDaemon_linux`、`devopsAgent_linux`、`upgrader_linux` +文件，其他系统依此类推。 - devopsDaemon: 用于守护agent进程，监控和拉起agent进程 - devopsAgent: 用于和调度服务通信，以及拉起构建进程worker @@ -29,32 +31,49 @@ windows编译，执行脚本 `build_windows.bat` - scripts/linux/stop.sh： agent停止脚本 - scripts/linux/uninstall.sh： agent卸载脚本 - ## 架构设计说明 + ### Agent 模块说明 + agent模块为GoAgent的核心模块。主要负责**执行worker以及做相关辅助工作**。 + #### 模块主流程说明 + ![image](https://github.com/Tencent/bk-ci/blob/master/docs/resource/img/Agent%E6%A8%A1%E5%9D%97%E6%B5%81%E7%A8%8B%E5%9B%BE.png) + #### 步骤说明 + 针对流程图步骤补充说明，序号为流程图步骤序号。 + ##### 1 检查进程。 -主要通过 **CheckProcess** 函数，函数主要通过输入的模块名称 例如：agent，获取runtime目录下的 name.pid 和 name.lock 文件。通过是否可以拿到 name.lock 文件锁的锁，来判断当前进程是否正在运行，如若没有在运行，则获取全局锁 total-lock.lock 来修改 name.pid 文件写入当前agent进程ID，同时一直持有 name.lock 锁直到进程退出时自动释放。 + +主要通过 **CheckProcess** 函数，函数主要通过输入的模块名称 例如：agent，获取runtime目录下的 name.pid 和 name.lock +文件。通过是否可以拿到 name.lock 文件锁的锁，来判断当前进程是否正在运行，如若没有在运行，则获取全局锁 total-lock.lock 来修改 +name.pid 文件写入当前agent进程ID，同时一直持有 name.lock 锁直到进程退出时自动释放。 ##### 2 初始化配置。 -1. 从 **.agent.properties** 文件中读取agent配置信息到 **GAgentConfig** 对象中。在读取的同时会写入一次，将 GAgentConfig 对象初始化的一些值持久化到 .agent.properties中。 + +1. 从 **.agent.properties** 文件中读取agent配置信息到 **GAgentConfig** 对象中。在读取的同时会写入一次，将 GAgentConfig + 对象初始化的一些值持久化到 .agent.properties中。 2. 初始化证书。 如果根目录中存在 **.cert** 文件，则将其添加至系统证书池，并添加至agent发送https请求的 **TLS** 配置中 -3. 初始化Agent环境变量。读取系统一些agent可能用到的环境变量到agent **GAgentEnv** 中，方便后续使用，例如 HostName，osName等。注：windows系统在这一步中因为可能持有虚拟网卡的IP，所以在上报agentIP地址时会 **忽略被配置了忽略虚拟网卡的windows机器IP**。 +3. 初始化Agent环境变量。读取系统一些agent可能用到的环境变量到agent **GAgentEnv** 中，方便后续使用，例如 + HostName，osName等。注：windows系统在这一步中因为可能持有虚拟网卡的IP，所以在上报agentIP地址时会 * + *忽略被配置了忽略虚拟网卡的windows机器IP**。 ##### 3 上报后台Agent启动。 + **POST|env服务|agent/thirdPartyAgent/startup** 并解析返回结果，如果成功返回则执行后续步骤否则退出。 ##### A 数据采集。 + 通过配置替换代码中写好的采集配置，启动 **telegraf** agent，并将数据上报到后台的influxDB。 ##### B 心跳上报。 + 无限循环，10s一次。通过 **POST|env服务|agent/thirdPartyAgent/agents/newHeartbeat** 发送agent环境的一些配置到后台，并解析后台结果，修改部分agent配置。 ##### C 检查升级。无限循环，20s一次。 + 1. 获取当前jdk版本信息 2. 通过 **POST|dispatch环境|agent/thirdPartyAgent/upgradeNew** 上报agent，worker，jdk版本信息判断是否需要升级。不需要升级则直接返回。 3. 下载最新版本应用对比md5判断是否需要升级。不需要升级则直接返回。 @@ -64,42 +83,62 @@ agent模块为GoAgent的核心模块。主要负责**执行worker以及做相关 7. 升级agent。执行 **upgrader** 模块。 ##### D pipeline。无限循环，30s一次。 + 1. 通过 **GET|env服务|agent/thirdPartyAgent/agents/pipelines** 获取需要执行的pipeline任务。 2. 通过 **PUT|env服务|agent/thirdPartyAgent/agents/pipelines** 更新任务状态。 3. 将任务脚本写入 工作空间下 **devops_pipeline_SeqId_type.sh** 文件。执行后获取输出内容，并通过更新任务状态上报后台，同时删除脚本文件。 ##### E 定期清理。 + 无限循环，2小时一次。删除超过配置中 **LogsKeepHours**的垃圾文件（hs_err_pid前缀文件）和日志文件。 ##### 4 从后台拉取构建任务。无限循环，5s一次。 + 1. 调用 **GET|env服务|agent/thirdPartyAgent/status** 获取agent状态，如果状态返回不正常则跳过这次任务。 2. 判断当前运行的instance是否超过配置 **ParallelTaskCount** 最大任务数，如果超过则跳过循环。 3. 获取**构建锁**，防止与其他任务差生冲突。 -4. 通过 **GET|dispatch服务|agent/thirdPartyAgent/startup** 获取构建任务，如果任务为空则跳过这次。否则则添加**预处理任务**到 **GBuildManager** 对象。注：unix构建机受**非登录用户启动进程无法拿到 ~/.bashrc 中的环境变量**的影响，需要创建 prestart和start脚本，通过设置 **exec -l** 来通过当前用户启动。 +4. 通过 **GET|dispatch服务|agent/thirdPartyAgent/startup** 获取构建任务，如果任务为空则跳过这次。否则则添加**预处理任务**到 + **GBuildManager** 对象。注：unix构建机受**非登录用户启动进程无法拿到 ~/.bashrc 中的环境变量**的影响，需要创建 + prestart和start脚本，通过设置 **exec -l** 来通过当前用户启动。 ##### 5 启动构建。 -1. 获取工作空间下的 **worker.jar** 文件。如果没有则尝试通过获取工作空间下的 **tmp** 目录下中同名文件进行复制使用来尝试自愈，如果 tmp 目录中的worker版本不对或者不存在文件，则进行 **结束构建**逻辑。 -2. 在工作空间下创建 **build_tmp** 目录，并在目录下启动 worker进程执行构建任务。任务启动后删除**预处理任务**同时添加**任务实例**到 **GBuildManager** 对象。 + +1. 获取工作空间下的 **worker.jar** 文件。如果没有则尝试通过获取工作空间下的 **tmp** 目录下中同名文件进行复制使用来尝试自愈，如果 + tmp 目录中的worker版本不对或者不存在文件，则进行 **结束构建**逻辑。 +2. 在工作空间下创建 **build_tmp** 目录，并在目录下启动 worker进程执行构建任务。任务启动后删除**预处理任务**同时添加**任务实例 + **到 **GBuildManager** 对象。 ##### F 等待进程结束 + 通过 **process.Wait()** 来等待进程结束，删除**GBuildManager** 对象中的任务实例，并执行 **结束构建**逻辑。 + ##### G 清理构建空间，即结束构建逻辑。 + - 清理构建过程中产生的文件脚本，与 build_tmp目录下超过7天的文件。 - 通过**POST|dispathc服务|agent/thirdPartyAgent/workerBuildFinish** 上报后台任务结束。 - - ### Daemon 模块说明 + daemon模块是agent模块的守护进程，主要功能是维护agent进程一直运行。 daemon模块在windows系统和unix系统中有不同的实现 + #### windows实现 + win实现主要是靠 **github.com/kardianos/service** 库调用的windows service功能实现，通过实现库中service接口维护agent一直运行。 + #### unix实现 -unix中的实现主要通过使用**定时器，5s一次**检查Agent模块进程是否存在，检测方式通过获取agent的文件锁，进程推出后文件锁会自动释放，所以如果无法获取文件锁，说明进程正常运行，如果可以获取文件锁，则说明agent进程退出了，这时Daemon会将其拉起。 + +unix中的实现主要通过使用**定时器，5s一次** +检查Agent模块进程是否存在，检测方式通过获取agent的文件锁，进程推出后文件锁会自动释放，所以如果无法获取文件锁，说明进程正常运行，如果可以获取文件锁，则说明agent进程退出了，这时Daemon会将其拉起。 ### Install & Upgrade 模块说明 + 安装和升级模块，主要负责daemon模块和agent模块的升级。 + #### install模块 + 安装模块功能较为简单，主要通过调用安装脚本，安装agent。 + #### upgrade模块 + 升级模块和agent模块中的升级Agent的逻辑一致，通过杀掉Agent进程后，替换Agent文件并启动来完成升级。 diff --git a/src/agent/agent/go.mod b/src/agent/agent/go.mod index 5d3ffeb3254..0a13800cefe 100644 --- a/src/agent/agent/go.mod +++ b/src/agent/agent/go.mod @@ -4,20 +4,19 @@ go 1.19 require ( github.com/TencentBlueKing/bk-ci/agentcommon v0.0.0-00010101000000-000000000000 - github.com/docker/docker v23.0.3+incompatible + github.com/docker/docker v24.0.9+incompatible github.com/gofrs/flock v0.8.1 // 1.24 以上的版本引入了memcall和memguard会导致 // 1、ulimit corefile被设置为0 https://github.com/ci-plugins/memguard/blob/master/core/init.go // 2、arm64 linux panic报错 https://github.com/awnumar/memguard/issues/144 github.com/influxdata/telegraf v1.24.4 - github.com/kardianos/service v1.2.2 github.com/nicksnyder/go-i18n/v2 v2.2.1 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.9.3 - golang.org/x/text v0.7.0 + golang.org/x/text v0.15.0 gopkg.in/ini.v1 v1.67.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect - gotest.tools/v3 v3.0.3 + gotest.tools/v3 v3.5.0 ) require ( @@ -32,9 +31,9 @@ require ( github.com/go-ole/go-ole v1.2.6 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/go-cmp v0.5.9 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/gosnmp/gosnmp v1.35.0 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/influxdata/toml v0.0.0-20190415235208-270119a8ce65 // indirect @@ -46,7 +45,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/naoina/go-stringutil v0.1.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0-rc2 // indirect + github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b // indirect github.com/philhofer/fwd v1.1.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20220216144756-c35f1ee13d7c // indirect @@ -57,17 +56,17 @@ require ( // 1.24 telegraf 必须跟随 3.22.9 版本，之上的版本windows编译不通过 github.com/shirou/gopsutil/v3 v3.22.9 // indirect github.com/sleepinggenius2/gosmi v0.4.4 // indirect - github.com/stretchr/objx v0.5.0 // indirect - github.com/stretchr/testify v1.8.1 // indirect + github.com/stretchr/objx v0.5.2 // indirect + github.com/stretchr/testify v1.9.0 // indirect github.com/tinylib/msgp v1.1.6 // indirect - github.com/tklauser/go-sysconf v0.3.11 // indirect - github.com/tklauser/numcpus v0.6.0 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect - github.com/yusufpapurcu/wmi v1.2.2 // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/sys v0.6.0 // indirect - google.golang.org/protobuf v1.28.1 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + golang.org/x/net v0.23.0 // indirect + golang.org/x/sys v0.20.0 + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) @@ -75,11 +74,17 @@ require ( require ( github.com/fsouza/go-dockerclient v1.9.7 github.com/gorilla/websocket v1.5.0 - golang.org/x/sync v0.1.0 + golang.org/x/sync v0.3.0 ) require ( - cloud.google.com/go/compute v1.10.0 // indirect + cloud.google.com/go v0.110.4 // indirect + cloud.google.com/go/bigquery v1.52.0 // indirect + cloud.google.com/go/compute v1.21.0 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + cloud.google.com/go/iam v1.1.1 // indirect + cloud.google.com/go/monitoring v1.15.1 // indirect + cloud.google.com/go/pubsub v1.32.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/aws/aws-sdk-go-v2 v1.17.1 // indirect github.com/aws/aws-sdk-go-v2/config v1.17.8 // indirect @@ -94,33 +99,41 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.17.4 // indirect github.com/aws/smithy-go v1.13.4 // indirect github.com/blues/jsonata-go v1.5.4 // indirect - github.com/containerd/containerd v1.6.18 // indirect + github.com/containerd/containerd v1.6.26 // indirect + github.com/docker/distribution v2.8.2+incompatible // indirect + github.com/go-logr/logr v1.4.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/google/s2a-go v0.1.4 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect github.com/klauspost/compress v1.15.11 // indirect github.com/kr/pretty v0.3.0 // indirect github.com/moby/patternmatcher v0.5.0 // indirect github.com/moby/sys/sequential v0.5.0 // indirect github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect github.com/morikuni/aec v1.0.0 // indirect - github.com/opencontainers/runc v1.1.5 // indirect + github.com/opencontainers/runc v1.1.12 // indirect + github.com/rickb777/date v1.14.2 // indirect + github.com/rickb777/plural v1.2.2 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect - go.opencensus.io v0.23.0 // indirect - golang.org/x/crypto v0.5.0 // indirect - golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect - golang.org/x/oauth2 v0.3.0 // indirect - golang.org/x/tools v0.1.12 // indirect - google.golang.org/api v0.100.0 // indirect + go.opencensus.io v0.24.0 // indirect + golang.org/x/crypto v0.23.0 // indirect + golang.org/x/mod v0.10.0 // indirect + golang.org/x/oauth2 v0.10.0 // indirect + golang.org/x/tools v0.9.1 // indirect + google.golang.org/api v0.126.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a // indirect - google.golang.org/grpc v1.50.1 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect + google.golang.org/grpc v1.58.3 // indirect ) require ( + github.com/capnspacehook/taskmaster v0.0.0-20210519235353-1629df7c85e9 github.com/docker/cli v23.0.1+incompatible - github.com/docker/distribution v2.8.2+incompatible // indirect github.com/docker/go-connections v0.4.0 + github.com/kardianos/service v1.2.2 + github.com/shirou/gopsutil/v4 v4.24.5 github.com/spf13/pflag v1.0.5 ) diff --git a/src/agent/agent/go.sum b/src/agent/agent/go.sum index 0ab43478a43..e4e2e0b2e08 100644 --- a/src/agent/agent/go.sum +++ b/src/agent/agent/go.sum @@ -19,26 +19,33 @@ cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKP cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.104.0 h1:gSmWO7DY1vOm0MVU6DNXM11BWHHsTUmsC5cv1fuW5X8= +cloud.google.com/go v0.110.4 h1:1JYyxKMN9hd5dR2MYTPWkGUgcoxVVhg0LKNKEo0qvmk= +cloud.google.com/go v0.110.4/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/bigquery v1.42.0 h1:JuTk8po4bCKRwObdT0zLb1K0BGkGHJdtgs2GK3j2Gws= +cloud.google.com/go/bigquery v1.52.0 h1:JKLNdxI0N+TIUWD6t9KN646X27N5dQWq9dZbbTWZ8hc= +cloud.google.com/go/bigquery v1.52.0/go.mod h1:3b/iXjRQGU4nKa87cXeg6/gogLjO8C6PmuM8i5Bi/u4= cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o= -cloud.google.com/go/compute v1.10.0 h1:aoLIYaA1fX3ywihqpBk2APQKOo20nXsp1GEZQbx5Jk4= -cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= +cloud.google.com/go/compute v1.21.0 h1:JNBsyXVoOoNJtTQcnEY5uYpZIbeCTYIeDe0Xh1bySMk= +cloud.google.com/go/compute v1.21.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/iam v0.5.0 h1:fz9X5zyTWBmamZsqvqZqD7khbifcZF/q+Z1J8pfhIUg= -cloud.google.com/go/monitoring v1.7.0 h1:zK12aN/jzLRzrRXopEOUaG6kvuF6WJmlv1mXn1L7a6E= +cloud.google.com/go/iam v1.1.1 h1:lW7fzj15aVIXYHREOqjRBV9PsH0Z6u8Y46a1YGvQP4Y= +cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= +cloud.google.com/go/monitoring v1.15.1 h1:65JhLMd+JiYnXr6j5Z63dUYCuOg770p8a/VC+gil/58= +cloud.google.com/go/monitoring v1.15.1/go.mod h1:lADlSAlFdbqQuwwpaImhsJXu1QSdd3ojypXrFSMr2rM= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/pubsub v1.26.0 h1:Y/HcMxVXgkUV2pYeLMUkclMg0ue6U0jVyI5xEARQ4zA= +cloud.google.com/go/pubsub v1.32.0 h1:JOEkgEYBuUTHSyHS4TcqOFuWr+vD6qO/imsFqShUCp4= +cloud.google.com/go/pubsub v1.32.0/go.mod h1:f+w71I33OMyxf9VpMVcZbnG5KSUkCOUHYpFd5U1GdRc= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= @@ -48,6 +55,7 @@ code.cloudfoundry.org/clock v1.0.0 h1:kFXWQM4bxYvdBw2X8BbBeXwQNgfoWv1vqAk2ZZyBN2 collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= collectd.org v0.5.0 h1:y4uFSAuOmeVhG3GCRa3/oH+ysePfO/+eGJNfd0Qa3d8= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20210715213245-6c3934b029d8 h1:V8krnnfGj4pV65YLUm3C0/8bl7V5Nry2Pwvy3ru/wLc= github.com/Azure/azure-amqp-common-go/v3 v3.2.3 h1:uDF62mbd9bypXWi19V1bN5NZEO84JqgmI5G73ibAmrk= github.com/Azure/azure-event-hubs-go/v3 v3.3.20 h1:LRAy00JlV5aDqd0LFXwfwFReYzl03CtH/kD91OHrT94= github.com/Azure/azure-kusto-go v0.8.0 h1:AeO6VBRGzB1BhmWeheSyN+WSrx+1wmhHm47vzptitdw= @@ -101,7 +109,7 @@ github.com/Mellanox/rdmamap v0.0.0-20191106181932-7c3c4763a6ee h1:atI/FFjXh6hIVl github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg= github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= -github.com/Microsoft/hcsshim v0.9.6 h1:VwnDOgLeoi2du6dAznfmspNqTiwczvjv4K7NxuY9jsY= +github.com/Microsoft/hcsshim v0.9.10 h1:TxXGNmcbQxBKVWvjvTocNb6jrPyeHlk5EiDhhgHgggs= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -132,12 +140,14 @@ github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8V github.com/aliyun/alibaba-cloud-sdk-go v1.61.1836 h1:HY697lY0w1HInRKBy/IO0lGmsMj3YSSM32SwG2eLsWM= github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9 h1:FXrPTd8Rdlc94dKccl7KPmdmIbVh/OjelJ8/vgMRzcQ= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= github.com/antchfx/jsonquery v1.3.0 h1:rftVBKEXpj8C9WVu+4mbqL5hd6nLz7/AbIvAQlq3D7o= github.com/antchfx/xmlquery v1.3.12 h1:6TMGpdjpO/P8VhjnaYPXuqT3qyJ/VsqoyNTmJzNBTQ4= github.com/antchfx/xpath v1.2.1 h1:qhp4EW6aCOVr5XIkT+l6LJ9ck/JsUH/yyauNgTQkBF8= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 h1:q4dksr6ICHXqG5hm0ZW5IHyeEJXoIJSOZeBLmWPNeIQ= +github.com/apache/arrow/go/v12 v12.0.0 h1:xtZE63VWl7qLdB0JObIXvvhGjoVNrQ9ciIHG2OK5cmc= github.com/apache/iotdb-client-go v0.12.2-0.20220722111104-cd17da295b46 h1:28HyUQcr8ZCyCAatR0gkf9PuLr52U2T+66tx5Th0nxI= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= @@ -223,6 +233,8 @@ github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+Wji github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/caio/go-tdigest v3.1.0+incompatible h1:uoVMJ3Q5lXmVLCCqaMGHLBWnbGoN6Lpu7OAUPR60cds= +github.com/capnspacehook/taskmaster v0.0.0-20210519235353-1629df7c85e9 h1:5jmtWADt5DzD8NnPxcqd1FzbFNZNfbJGNeDb+WKjoJ0= +github.com/capnspacehook/taskmaster v0.0.0-20210519235353-1629df7c85e9/go.mod h1:257CYs3Wd/CTlLQ3c72jKv+fFE2MV3WPNnV5jiroYUU= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= @@ -232,12 +244,10 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= -github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= github.com/cisco-ie/nx-telemetry-proto v0.0.0-20220628142927-f4160bcb943c h1:k3y2XtIffIk230a+e0d7vbs5ebTvH3OcCMKN/jS6IAY= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -245,19 +255,21 @@ github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.6.18 h1:qZbsLvmyu+Vlty0/Ex5xc0z2YtKpIsb5n45mAMI+2Ns= -github.com/containerd/containerd v1.6.18/go.mod h1:1RdCUu95+gc2v9t3IL+zIlpClSmew7/0YS8O5eQZrOw= -github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= +github.com/containerd/containerd v1.6.26 h1:VVfrE6ZpyisvB1fzoY8Vkiq4sy+i5oF4uk7zu03RaHs= +github.com/containerd/containerd v1.6.26/go.mod h1:I4TRdsdoo5MlKob5khDJS2EPT1l1oMNaE2MBm6FrwxM= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/coocood/freecache v1.2.2 h1:UPkJCxhRujykq1jXuwxAPgDHnm6lKGrLZPnuHzgWRtE= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/couchbase/go-couchbase v0.1.1 h1:ClFXELcKj/ojyoTYbsY34QUrrYCBi/1G749sXSCkdhk= github.com/couchbase/gomemcached v0.1.3 h1:HIc5qMYNbuhB7zNaiEtj61DCYkquAwrQlf64q7JzdEY= @@ -267,7 +279,6 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -288,8 +299,8 @@ github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4Kfc github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v20.10.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v23.0.3+incompatible h1:9GhVsShNWz1hO//9BNg/dpMnZW25KydO4wtVxWAIbho= -github.com/docker/docker v23.0.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v24.0.9+incompatible h1:HPGzNmwfLZWdxHqK9/II92pyi1EpYKsAqcl4G0Of9v0= +github.com/docker/docker v24.0.9+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= @@ -312,14 +323,15 @@ github.com/eclipse/paho.mqtt.golang v1.4.1 h1:tUSpviiL5G3P9SZZJPC4ZULZJsxQKXxfEN github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk= -github.com/emicklei/go-restful/v3 v3.8.0 h1:eCZ8ulSerjdAiaNpF7GxXIE7ZCMo1moN1qX+S609eVw= +github.com/emicklei/go-restful v2.16.0+incompatible h1:rgqiKNjTnFQA6kkhFe16D8epTksy9HQ1MyrbDXSdYhM= +github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -331,7 +343,6 @@ github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoD github.com/form3tech-oss/jwt-go v3.2.5+incompatible h1:/l4kBbb4/vGSsdtB5nUe8L7B9mImVMaBPw9L/0TBHU8= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= -github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsouza/go-dockerclient v1.9.7 h1:FlIrT71E62zwKgRvCvWGdxRD+a/pIy+miY/n3MXgfuw= github.com/fsouza/go-dockerclient v1.9.7/go.mod h1:vx9C32kE2D15yDSOMCDaAEIARZpDQDFBHeqL3MgQy/U= @@ -355,7 +366,9 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= @@ -479,8 +492,7 @@ github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/V github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= @@ -527,8 +539,9 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -537,7 +550,7 @@ github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/flatbuffers v2.0.0+incompatible h1:dicJ2oXwypfwUGnB2/TYWYEKiuk9eYQlQO/AnOHl5mI= +github.com/google/flatbuffers v2.0.8+incompatible h1:ivUb1cGomAB101ZM1T0nOiWz9pSrTMoa9+EiY7igmkM= github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= github.com/google/gnxi v0.0.0-20221016143401-2aeceb5a2901 h1:xlsMG0I0F6Ou3a4zRWu3cThivTt2N2V1cZafIloTBTU= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -553,8 +566,9 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github/v32 v32.1.0 h1:GWkQOdXqviCPx7Q7Fj+KyPoGm4SwHRh8rheoPhd27II= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= @@ -577,17 +591,19 @@ github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210323184331-8eee2492667d/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= +github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs= -github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= +github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= +github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.6.0 h1:SXk3ABtQYDT/OH8jAyvEOQ58mgawq5C4o/4/89qN2ZU= +github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/gopcua/opcua v0.3.7 h1:iGjLW3D+ztnjtZQPKsJ0nwibHyDw1m11NfqOU8KSFQ8= github.com/gophercloud/gophercloud v0.16.0/go.mod h1:wRtmUelyIIv3CSSDI47aUwbs075O6i+LY+pXsKCBsb4= @@ -738,11 +754,14 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c= github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5 h1:2U0HzY8BJ8hVwDKIzp7y4voR9CX/nvcfymLmg2UiOio= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kolo/xmlrpc v0.0.0-20201022064351-38db28db192b h1:iNjcivnc6lhbvJA3LD622NPrUponluJrBWPIwGG/3Bg= @@ -752,7 +771,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -813,6 +831,8 @@ github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3N github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= @@ -837,7 +857,6 @@ github.com/moby/ipvs v1.0.2 h1:NSbzuRTvfneftLU3VwPU5QuA6NZ0IUmqq9+VHcQxqHw= github.com/moby/patternmatcher v0.5.0 h1:YCZgJOeULcxLw1Q+sVR636pmS7sPEn1Qo2iAN6M7DBo= github.com/moby/patternmatcher v0.5.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= @@ -853,7 +872,6 @@ github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6f github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/multiplay/go-ts3 v1.0.1 h1:Ja8ho7UzUDNvNCwcDzPEPimLRub7MUqbD+sgMWkcR0A= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= @@ -881,6 +899,7 @@ github.com/nicksnyder/go-i18n/v2 v2.2.1 h1:aOzRCdwsJuoExfZhoiXHy4bjruwCMdt5otbYo github.com/nicksnyder/go-i18n/v2 v2.2.1/go.mod h1:fF2++lPHlo+/kPaj3nB0uxtPwzlPm+BlgwGX7MkeGj0= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nsqio/go-nsq v1.1.0 h1:PQg+xxiUjA7V+TLdXw7nVrJ5Jbl3sN86EhGCQj4+FYE= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= @@ -891,20 +910,22 @@ github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.2 h1:aY/nuoWlKJud2J6U0E3NWsjlg+0GtwXxgEqthRdzlcs= +github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/openconfig/gnmi v0.0.0-20220920173703-480bf53a74d2 h1:3YLlQFLDsFTvruKoYBbuYqhCgsXMtNewSrLjNXcF/Sg= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= -github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= -github.com/opencontainers/runc v1.1.5 h1:L44KXEpKmfWDcS02aeGm8QNTFXTo2D+8MYGDIJ/GDEs= -github.com/opencontainers/runc v1.1.5/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= -github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= +github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b h1:YWuSjZCQAPM8UUBLkYUk1e+rZcvWHJmFb6i6rM44Xs8= +github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= +github.com/opencontainers/runc v1.1.12 h1:BOIssBaW1La0/qbNZHXOOa71dZfZEQOzW7dqQf3phss= +github.com/opencontainers/runc v1.1.12/go.mod h1:S+lQwSfncpBha7XTy/5lBwWgm5+y5Ma/O44Ekby9FK8= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing-contrib/go-stdlib v1.0.0/go.mod h1:qtI1ogk+2JhVPIXVc6q+NHziSmy2W5GbdQZFUHADCBU= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= @@ -997,6 +1018,10 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqn github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= +github.com/rickb777/date v1.14.2 h1:PCme7ZL/cniZmDgS9Pyn5fHmu5A6lz12Ibfd33FmDiw= +github.com/rickb777/date v1.14.2/go.mod h1:swmf05C+hN+m8/Xh7gEq3uB6QJDNc5pQBWojKdHetOs= +github.com/rickb777/plural v1.2.2 h1:4CU5NiUqXSM++2+7JCrX+oguXd2D7RY5O1YisMw1yCI= +github.com/rickb777/plural v1.2.2/go.mod h1:xyHbelv4YvJE51gjMnHvk+U2e9zIysg6lTnSQK8XUYA= github.com/riemann/riemann-go-client v0.5.1-0.20211206220514-f58f10cdce16 h1:bGXoxRwUpPTCaQ86DRE+3wqE9vh3aH8W0HH5L/ygOFM= github.com/robbiet480/go.nut v0.0.0-20220219091450-bd8f121e1fa1 h1:YmFqprZILGlF/X3tvMA4Rwn3ySxyE3hGUajBHkkaZbM= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= @@ -1017,12 +1042,16 @@ github.com/samuel/go-zookeeper v0.0.0-20200724154423-2164a8ac840e h1:CGjiMQ0wMH4 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210223165440-c65ae3540d44/go.mod h1:CJJ5VAbozOl0yEw7nHB9+7BXTJbIn6h7W+f6Gau5IP8= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shirou/gopsutil/v3 v3.22.9 h1:yibtJhIVEMcdw+tCTbOPiF1VcsuDeTE4utJ8Dm4c5eA= github.com/shirou/gopsutil/v3 v3.22.9/go.mod h1:bBYl1kjgEJpWpxeHmLI+dVHWtyAwfcmSBLDsp2TNT8A= +github.com/shirou/gopsutil/v4 v4.24.5 h1:gGsArG5K6vmsh5hcFOHaPm87UD003CaDMkAOweSQjhM= +github.com/shirou/gopsutil/v4 v4.24.5/go.mod h1:aoebb2vxetJ/yIDZISmduFvVNPHqXQ9SEJwRXxkf0RA= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/showwin/speedtest-go v1.2.1 h1:5GrQFGn5N4YRBCaiph6ay6Py9yL2k7Ja10bbUZl9HPE= github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= @@ -1037,7 +1066,6 @@ github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sleepinggenius2/gosmi v0.4.4 h1:xgu+Mt7CptuB10IPt3SVXBAA9tARToT4B9xGzjjxQX8= @@ -1063,8 +1091,9 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -1074,9 +1103,9 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/testcontainers/testcontainers-go v0.14.0 h1:h0D5GaYG9mhOWr2qHdEKDXpkce/VlvaYOCzTRi6UBi8= github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= @@ -1086,11 +1115,11 @@ github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDW github.com/tinylib/msgp v1.1.6 h1:i+SbKraHhnrf9M5MYmvQhFnbLhAXSDWF8WWsuyRdocw= github.com/tinylib/msgp v1.1.6/go.mod h1:75BAfg2hauQhs3qedfdDZmWAPcFMAvJE5b9rGOMufyw= github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk= -github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= -github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ= -github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= -github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= @@ -1100,9 +1129,7 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vapourismo/knx-go v0.0.0-20220829185957-fb5458a5389d h1:BJMc7MNW/p80cCkC46JimNuowOWCnSSW5IHjtUrXzNk= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= -github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs= -github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vishvananda/netns v0.0.0-20220913150850-18c4f4234207 h1:nn7SOQy8xCu3iXNv7oiBhhEQtbWdnEOMnuKBlHvrqIM= github.com/vjeantet/grok v1.0.1 h1:2rhIR7J4gThTgcZ1m2JY4TrJZNgjn985U28kT2wQrJ4= github.com/vmware/govmomi v0.28.1-0.20220921224932-b4b508abf208 h1:IDVzGQ2aczmTEfTos4hzmFw20tGQ4zZsVnel9C6VEpA= @@ -1133,8 +1160,10 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da h1:NimzV1aGyq29m5ukMK0AMWEhFaL/lrEOaephfuoiARg= -github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= @@ -1154,9 +1183,11 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/collector/pdata v0.63.0 h1:YPeMzF4OYFeMW6E+A/eQEv5s32wpc5wEa24H2PP5LeE= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -1189,8 +1220,9 @@ golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= -golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= +golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1228,8 +1260,9 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1264,11 +1297,13 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -1279,9 +1314,10 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210324051636-2c4c8ecb7826/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1294,8 +1330,8 @@ golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210323180902-22b0adad7558/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.3.0 h1:6l90koy8/LaBLmLu8jpHeHexzMwEita0zFfYlggy2F8= -golang.org/x/oauth2 v0.3.0/go.mod h1:rQrIauxkUhJ6CuwEXwymO2/eh4xz2ZWF1nBkcxS+tGk= +golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= +golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1309,8 +1345,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1331,18 +1367,18 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1384,25 +1420,24 @@ golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210314195730-07df6a141424/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= +golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1410,16 +1445,18 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220922220347-f3bd1da661af h1:Yx9k8YCG3dvF87UAn2tu2HQLf2dt/eR1bXxpLMWeH+Y= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1490,8 +1527,9 @@ golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= +golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1527,8 +1565,8 @@ google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34q google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= google.golang.org/api v0.42.0/go.mod h1:+Oj4s6ch2SEGtPjGqfUfZonBH0GjQH89gTeKKAEGZKI= -google.golang.org/api v0.100.0 h1:LGUYIrbW9pzYQQ8NWXlaIVkgnfubVBZbMFb9P8TK374= -google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= +google.golang.org/api v0.126.0 h1:q4GJq+cAdMAC7XP7njvQ4tvohGLiSlytuL4BQxbIZ+o= +google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1580,8 +1618,10 @@ google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210312152112-fc591d9ea70f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a h1:GH6UPn3ixhWcKDhpnEC55S75cerLPdpp3hrhfKYjZgw= -google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= +google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 h1:Z0hjGZePRE0ZBWotvtrwxFNrNE9CUAGtplaDK5NNI/g= +google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 h1:FmF5cCW94Ij59cfpoLiwTgodWmm60eEV0CjlsVg2fuw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= @@ -1605,8 +1645,9 @@ google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.50.1 h1:DS/BukOZWp8s6p4Dt/tOaJaTQyPyOoCcrjroHuCeLzY= -google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= +google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1619,9 +1660,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1665,8 +1705,9 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= -gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= +gotest.tools/v3 v3.5.0 h1:Ljk6PdHdOhAb5aDMWXjDLMMhph+BpztA4v1QdqEW2eY= +gotest.tools/v3 v3.5.0/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/src/agent/agent/internal/third_party/dep/fs/fs.go b/src/agent/agent/internal/third_party/dep/fs/fs.go index 69cb76efe2c..3ed8ee66d59 100644 --- a/src/agent/agent/internal/third_party/dep/fs/fs.go +++ b/src/agent/agent/internal/third_party/dep/fs/fs.go @@ -32,6 +32,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package fs import ( + "fmt" "github.com/pkg/errors" "io" "os" @@ -41,7 +42,7 @@ import ( ) // fs contains a copy of a few functions from dep tool code to avoid a dependency on golang/dep. -// This code is copied from https://github.com/golang/dep/blob/a5440af88cd9b4507810256f8845a297936868a2/internal/fs/fs.go +// This code is copied from https://github.com/golang/dep/blob/master/internal/fs/fs.go // No changes to the code were made other than removing some unused functions // RenameWithFallback attempts to rename a file or directory, but falls back to @@ -50,7 +51,7 @@ import ( func RenameWithFallback(src, dst string) error { _, err := os.Stat(src) if err != nil { - return errors.Wrapf(err, "cannot stat %s", src) + return errors.Wrapf(err, "cannot stat %s error", src) } err = os.Rename(src, dst) @@ -91,7 +92,7 @@ func IsDir(name string) (bool, error) { return false, err } if !fi.IsDir() { - return false, errors.Errorf("%q is not a directory", name) + return false, fmt.Errorf("%q is not a directory", name) } return true, nil } diff --git a/src/agent/agent/internal/third_party/dep/fs/rename.go b/src/agent/agent/internal/third_party/dep/fs/rename.go index 277e9b72973..f9b79c89ede 100644 --- a/src/agent/agent/internal/third_party/dep/fs/rename.go +++ b/src/agent/agent/internal/third_party/dep/fs/rename.go @@ -35,10 +35,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package fs import ( + "github.com/pkg/errors" "os" "syscall" - - "github.com/pkg/errors" ) // renameFallback attempts to determine the appropriate fallback to failed rename diff --git a/src/agent/agent/internal/third_party/dep/fs/rename_windows.go b/src/agent/agent/internal/third_party/dep/fs/rename_windows.go index d8207cc5f7a..bc4ef9e4d43 100644 --- a/src/agent/agent/internal/third_party/dep/fs/rename_windows.go +++ b/src/agent/agent/internal/third_party/dep/fs/rename_windows.go @@ -35,10 +35,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. package fs import ( + "github.com/pkg/errors" "os" "syscall" - - "github.com/pkg/errors" ) // renameFallback attempts to determine the appropriate fallback to failed rename diff --git a/src/agent/agent/src/cmd/daemon/main.go b/src/agent/agent/src/cmd/daemon/main.go index 248e7980bb9..c253eda5a08 100644 --- a/src/agent/agent/src/cmd/daemon/main.go +++ b/src/agent/agent/src/cmd/daemon/main.go @@ -32,6 +32,7 @@ package main import ( "fmt" + "github.com/pkg/errors" "io" "os" "os/exec" @@ -102,6 +103,7 @@ func main() { logs.Info("devops daemon start") logs.Info("pid: ", os.Getpid()) + logs.Info("workDir: ", workDir) watch(isDebug) systemutil.KeepProcessAlive() @@ -120,7 +122,7 @@ func watch(isDebug bool) { select { case <-checkTimeTicker.C: if err := totalLock.Lock(); err != nil { - logs.Errorf("failed to get agent lock: %v", err) + logs.WithError(err).Error("failed to get agent lock") continue } @@ -139,12 +141,12 @@ func doCheckAndLaunchAgent(isDebug bool) { defer func() { err = agentLock.Unlock() if err != nil { - logs.Error("try to unlock agent.lock failed", err) + logs.WithError(err).Error("try to unlock agent.lock failed") } }() } if err != nil { - logs.Errorf("try to get agent.lock failed: %v", err) + logs.WithError(err).Error("try to get agent.lock failed") return } if !locked { @@ -155,7 +157,7 @@ func doCheckAndLaunchAgent(isDebug bool) { process, err := launch(workDir+"/"+config.AgentFileClientLinux, isDebug) if err != nil { - logs.Errorf("launch agent failed: %v", err) + logs.WithError(err).Error("launch agent failed") return } if process == nil { @@ -182,38 +184,38 @@ func launch(agentPath string, isDebug bool) (*os.Process, error) { err := fileutil.SetExecutable(agentPath) if err != nil { - return nil, fmt.Errorf("chmod agent file failed: %v", err) + return nil, errors.Wrap(err, "chmod agent file failed") } // 获取 agent 的错误输出，这样有助于打印出崩溃的堆栈方便排查问题 stdErr, errstd := cmd.StderrPipe() if errstd != nil { - logs.Error("get agent stderr pipe error", errstd) + logs.WithError(errstd).Error("get agent stderr pipe error") } if err = cmd.Start(); err != nil { if stdErr != nil { stdErr.Close() } - return nil, fmt.Errorf("start agent failed: %v", err) + return nil, errors.Wrap(err, "start agent failed") } go func() { if err := cmd.Wait(); err != nil { if exiterr, ok := err.(*exec.ExitError); ok { - if exiterr.ExitCode() == constant.DAEMON_EXIT_CODE { - logs.Warnf("exit code %d daemon exit", constant.DAEMON_EXIT_CODE) - systemutil.ExitProcess(constant.DAEMON_EXIT_CODE) + if exiterr.ExitCode() == constant.DaemonExitCode { + logs.Warnf("exit code %d daemon exit", constant.DaemonExitCode) + systemutil.ExitProcess(constant.DaemonExitCode) } } - logs.Error("agent process error", err) + logs.WithError(err).Error("agent process error") if errstd != nil { return } defer stdErr.Close() out, err := io.ReadAll(stdErr) if err != nil { - logs.Error("read agent stderr out error", err) + logs.WithError(err).Error("read agent stderr out error") return } logs.Error("agent process error out", string(out)) diff --git a/src/agent/agent/src/cmd/daemon/main_win.go b/src/agent/agent/src/cmd/daemon/main_win.go index f57da577b28..5e0b710a6be 100644 --- a/src/agent/agent/src/cmd/daemon/main_win.go +++ b/src/agent/agent/src/cmd/daemon/main_win.go @@ -31,6 +31,7 @@ package main import ( + "errors" "fmt" "io" "os" @@ -68,8 +69,8 @@ func main() { } // 初始化日志 - logFilePath := filepath.Join(systemutil.GetWorkDir(), "logs", "devopsDaemon.log") - err := logs.Init(logFilePath, isDebug, false) + workDir := systemutil.GetExecutableDir() + err := logs.Init(filepath.Join(workDir, "logs", "devopsDaemon.log"), isDebug, false) if err != nil { fmt.Printf("init daemon log error %v\n", err) systemutil.ExitProcess(1) @@ -77,7 +78,6 @@ func main() { logs.Infof("GOOS=%s, GOARCH=%s", runtime.GOOS, runtime.GOARCH) - workDir := systemutil.GetExecutableDir() err = os.Chdir(workDir) if err != nil { logs.Info("change work dir failed, err: ", err.Error()) @@ -91,34 +91,31 @@ func main() { } }() + if ok := systemutil.CheckProcess(daemonProcess); !ok { + logs.Info("get process lock failed, exit") + return + } + logs.Info("devops daemon start") logs.Info("pid: ", os.Getpid()) logs.Info("workDir: ", workDir) //服务定义 serviceConfig := &service.Config{ - Name: "name", - DisplayName: "displayName", - Description: "description", - WorkingDirectory: "C:/data/landun", - } - - if ok := systemutil.CheckProcess(daemonProcess); !ok { - logs.Info("get process lock failed, exit") - return + Name: "name", } daemonProgram := &program{} sys := service.ChosenSystem() daemonService, err := sys.New(daemonProgram, serviceConfig) if err != nil { - logs.Error("Init service error: ", err.Error()) + logs.WithError(err).Error("Init service error") systemutil.ExitProcess(1) } err = daemonService.Run() if err != nil { - logs.Error("run agent program error: ", err.Error()) + logs.WithError(err).Error("run agent program error") } } @@ -128,65 +125,70 @@ func watch() { workDir := systemutil.GetExecutableDir() var agentPath = systemutil.GetWorkDir() + "/devopsAgent.exe" for { - cmd := exec.Command(agentPath) - cmd.Dir = workDir - - // 获取 agent 的错误输出，这样有助于打印出崩溃的堆栈方便排查问题 - stdErr, errstd := cmd.StderrPipe() - if errstd != nil { - logs.Error("get agent stderr pipe error", errstd) - } else { - defer stdErr.Close() - } - - logs.Info("start devops agent") - if !fileutil.Exists(agentPath) { - logs.Error("agent file: ", agentPath, " not exists") - logs.Info("restart after 30 seconds") - time.Sleep(30 * time.Second) - } - - err := fileutil.SetExecutable(agentPath) - if err != nil { - logs.Error("chmod failed, err: ", err.Error()) - logs.Info("restart after 30 seconds") - time.Sleep(30 * time.Second) - continue - } + func() { + cmd := exec.Command(agentPath) + cmd.Dir = workDir - err = cmd.Start() - if err != nil { - logs.Error("agent start failed, err: ", err.Error()) - logs.Info("restart after 30 seconds") - time.Sleep(30 * time.Second) - continue - } + // 获取 agent 的错误输出，这样有助于打印出崩溃的堆栈方便排查问题 + stdErr, errstd := cmd.StderrPipe() + if errstd != nil { + logs.WithError(errstd).Error("get agent stderr pipe error") + } else { + defer stdErr.Close() + } - GAgentProcess = cmd.Process - logs.Info("devops agent started, pid: ", cmd.Process.Pid) - _, err = cmd.Process.Wait() - if err != nil { - if exiterr, ok := err.(*exec.ExitError); ok { - if exiterr.ExitCode() == constant.DAEMON_EXIT_CODE { - logs.Warnf("exit code %d daemon exit", constant.DAEMON_EXIT_CODE) - systemutil.ExitProcess(constant.DAEMON_EXIT_CODE) - } + logs.Info("start devops agent") + if !fileutil.Exists(agentPath) { + logs.Errorf("agent file: %s not exists", agentPath) + logs.Info("restart after 30 seconds") + time.Sleep(30 * time.Second) } - logs.Error("agent process error", err) - if errstd != nil { + + err := fileutil.SetExecutable(agentPath) + if err != nil { + logs.WithError(err).Error("chmod failed, err") + logs.Info("restart after 30 seconds") + time.Sleep(30 * time.Second) return } - out, err := io.ReadAll(stdErr) + + err = cmd.Start() if err != nil { - logs.Error("read agent stderr out error", err) + logs.WithError(err).Error("agent start failed, err") + logs.Info("restart after 30 seconds") + time.Sleep(30 * time.Second) return } - logs.Error("agent process error out", string(out)) - } - logs.Info("agent process exited") - logs.Info("restart after 30 seconds") - time.Sleep(30 * time.Second) + GAgentProcess = cmd.Process + logs.Info("devops agent started, pid: ", cmd.Process.Pid) + err = cmd.Wait() + if err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + if exitErr.ExitCode() == constant.DaemonExitCode { + logs.Warnf("exit code %d daemon exit", constant.DaemonExitCode) + systemutil.ExitProcess(constant.DaemonExitCode) + } + } + logs.WithError(err).Error("agent process error") + + // 读取可能的报错 + if errstd != nil { + return + } + out, err := io.ReadAll(stdErr) + if err != nil { + logs.WithError(err).Error("read agent stderr out error") + } else { + logs.Error("agent process error out", string(out)) + } + } + logs.Info("agent process exited") + + logs.Info("restart after 30 seconds") + time.Sleep(30 * time.Second) + }() } } diff --git a/src/agent/agent/src/cmd/installer/main.go b/src/agent/agent/src/cmd/installer/main.go index cf50314137f..db06be8b92d 100644 --- a/src/agent/agent/src/cmd/installer/main.go +++ b/src/agent/agent/src/cmd/installer/main.go @@ -76,7 +76,7 @@ func main() { if config.ActionInstall == *action { err := installer.DoInstallAgent() if err != nil { - logs.Error("install new agent failed: " + err.Error()) + logs.WithError(err).Error("install new agent failed") systemutil.ExitProcess(1) } } else if config.ActionUninstall == *action { diff --git a/src/agent/agent/src/cmd/translation_generator/translation_generator.go b/src/agent/agent/src/cmd/translation_generator/translation_generator.go index 3baf915f104..8357a3685a1 100644 --- a/src/agent/agent/src/cmd/translation_generator/translation_generator.go +++ b/src/agent/agent/src/cmd/translation_generator/translation_generator.go @@ -31,6 +31,7 @@ import ( "bytes" "encoding/json" "fmt" + "github.com/TencentBlueKing/bk-ci/agent/src/pkg/constant" "go/format" "os" "path/filepath" @@ -148,14 +149,14 @@ func main() { src := g.format() // 写入到 translation.go 文件中 outputName := filepath.Join(workDir, "translation", "translation.go") - err = os.WriteFile(outputName, src, 0644) + err = os.WriteFile(outputName, src, constant.CommonFileModePerm) if err != nil { fmt.Fprintf(os.Stderr, "writing output: %s", err) exit() } } -// 生成器保存分析的状态。 主要用来缓冲 format.Source 的输出。 +// Generator 生成器保存分析的状态。 主要用来缓冲 format.Source 的输出。 type Generator struct { buf bytes.Buffer // 累计输出 } diff --git a/src/agent/agent/src/cmd/upgrader/main.go b/src/agent/agent/src/cmd/upgrader/main.go index fea9f816e20..6e32c99d89b 100644 --- a/src/agent/agent/src/cmd/upgrader/main.go +++ b/src/agent/agent/src/cmd/upgrader/main.go @@ -74,7 +74,7 @@ func main() { if config.ActionUpgrade == *action { err := upgrader.DoUpgradeAgent() if err != nil { - logs.Error("upgrade agent failed: " + err.Error()) + logs.WithError(err).Error("upgrade agent failed") systemutil.ExitProcess(1) } } else if config.ActionUninstall == *action { diff --git a/src/agent/agent/src/pkg/agent/agent.go b/src/agent/agent/src/pkg/agent/agent.go index d72ce2240ea..b980f4d4aa7 100644 --- a/src/agent/agent/src/pkg/agent/agent.go +++ b/src/agent/agent/src/pkg/agent/agent.go @@ -51,13 +51,23 @@ func Run(isDebug bool) { // 初始化国际化 i18n.InitAgentI18n() + // 启动 agent，需要等到上报启动成功才能继续 _, err := job.AgentStartup() if err != nil { - logs.Warn("agent startup failed: ", err.Error()) + logs.WithError(err).Error("agent startup failed") + for { + _, err = job.AgentStartup() + if err == nil { + break + } else { + logs.WithError(err).Error("agent startup failed") + time.Sleep(5 * time.Second) + } + } } // 数据采集 - go collector.DoAgentCollect() + go collector.Collect() // 定期清理 go cron.CleanJob() @@ -102,7 +112,7 @@ func doAsk() { } if err != nil { - logs.Error("ask request failed: ", err.Error()) + logs.WithErrorNoStack(err).Error("ask request failed") return } if result.IsNotOk() { @@ -122,7 +132,7 @@ func doAsk() { resp := new(api.AskResp) err = util.ParseJsonToData(result.Data, &resp) if err != nil { - logs.Error("parse ask resp failed: ", err.Error()) + logs.WithErrorNoStack(err).Error("parse ask resp failed") return } diff --git a/src/agent/agent/src/pkg/agent/ask.go b/src/agent/agent/src/pkg/agent/ask.go index 3fc95ea2287..b80a0fe4790 100644 --- a/src/agent/agent/src/pkg/agent/ask.go +++ b/src/agent/agent/src/pkg/agent/ask.go @@ -53,7 +53,7 @@ func genHeartInfoAndUpgrade( MasterVersion: config.AgentVersion, SlaveVersion: config.GAgentEnv.SlaveVersion, HostName: config.GAgentEnv.HostName, - AgentIp: config.GAgentEnv.AgentIp, + AgentIp: config.GAgentEnv.GetAgentIp(), ParallelTaskCount: config.GAgentConfig.ParallelTaskCount, AgentInstallPath: systemutil.GetExecutableDir(), StartedUser: systemutil.GetCurrentUser().Username, diff --git a/src/agent/agent/src/pkg/agent/heartbeat.go b/src/agent/agent/src/pkg/agent/heartbeat.go index df76588e8fa..1b35bbb7a22 100644 --- a/src/agent/agent/src/pkg/agent/heartbeat.go +++ b/src/agent/agent/src/pkg/agent/heartbeat.go @@ -85,7 +85,23 @@ func agentHeartbeat(heartbeatResponse *api.AgentHeartbeatResponse) { } // agent环境变量 - config.GEnvVars = heartbeatResponse.Envs + if heartbeatResponse.Envs != nil { + if config.GApiEnvVars.Size() <= 0 { + config.GApiEnvVars.SetEnvs(heartbeatResponse.Envs) + } else { + flag := false + config.GApiEnvVars.RangeDo(func(k, v string) bool { + if heartbeatResponse.Envs[k] != v { + flag = true + return false + } + return true + }) + if flag { + config.GApiEnvVars.SetEnvs(heartbeatResponse.Envs) + } + } + } /* 忽略一些在Windows机器上VPN代理软件所产生的虚拟网卡（有Mac地址）的IP，一般这类IP @@ -93,8 +109,8 @@ func agentHeartbeat(heartbeatResponse *api.AgentHeartbeatResponse) { */ if len(config.GAgentConfig.IgnoreLocalIps) > 0 { splitIps := util.SplitAndTrimSpace(config.GAgentConfig.IgnoreLocalIps, ",") - if util.Contains(splitIps, config.GAgentEnv.AgentIp) { // Agent检测到的IP与要忽略的本地VPN IP相同，则更换真正IP - config.GAgentEnv.AgentIp = systemutil.GetAgentIp(splitIps) + if util.Contains(splitIps, config.GAgentEnv.GetAgentIp()) { // Agent检测到的IP与要忽略的本地VPN IP相同，则更换真正IP + config.GAgentEnv.SetAgentIp(systemutil.GetAgentIp(splitIps)) } } diff --git a/src/agent/agent/src/pkg/api/api.go b/src/agent/agent/src/pkg/api/api.go index 45a4d8d0396..254d515aee5 100644 --- a/src/agent/agent/src/pkg/api/api.go +++ b/src/agent/agent/src/pkg/api/api.go @@ -61,7 +61,7 @@ func AgentStartup() (*httputil.DevopsResult, error) { startInfo := &ThirdPartyAgentStartInfo{ HostName: config.GAgentEnv.HostName, - HostIp: config.GAgentEnv.AgentIp, + HostIp: config.GAgentEnv.GetAgentIp(), DetectOs: config.GAgentEnv.OsName, MasterVersion: config.AgentVersion, SlaveVersion: config.GAgentEnv.SlaveVersion, diff --git a/src/agent/agent/src/pkg/api/type.go b/src/agent/agent/src/pkg/api/type.go index 005b23605f3..f46659fed89 100644 --- a/src/agent/agent/src/pkg/api/type.go +++ b/src/agent/agent/src/pkg/api/type.go @@ -85,9 +85,10 @@ type Credential struct { } type DockerOptions struct { - Volumes []string `json:"volumes"` - Gpus string `json:"gpus"` - Mounts []string `json:"mounts"` + Volumes []string `json:"volumes"` + Gpus string `json:"gpus"` + Mounts []string `json:"mounts"` + Privileged bool `json:"privileged"` } type ThirdPartyBuildWithStatus struct { diff --git a/src/agent/agent/src/pkg/collector/collector.go b/src/agent/agent/src/pkg/collector/collector.go index 0caf71911d0..dd2e528cae5 100644 --- a/src/agent/agent/src/pkg/collector/collector.go +++ b/src/agent/agent/src/pkg/collector/collector.go @@ -31,14 +31,12 @@ import ( "bytes" "context" "fmt" - "os" + "github.com/pkg/errors" "text/template" "time" telegrafconf "github.com/TencentBlueKing/bk-ci/agent/src/pkg/collector/telegrafConf" "github.com/TencentBlueKing/bk-ci/agentcommon/utils/fileutil" - "github.com/pkg/errors" - "github.com/influxdata/telegraf/logger" "github.com/TencentBlueKing/bk-ci/agent/src/pkg/util/systemutil" @@ -54,63 +52,80 @@ import ( ) const ( - telegrafConfigFile = "telegraf.conf" telegrafRelaunchTime = 5 * time.Second - - templateKeyAgentId = "###{agentId}###" - templateKeyAgentSecret = "###{agentSecret}###" - templateKeyGateway = "###{gateway}###" - templateKeyTlsCa = "###{tls_ca}###" - templateKeyProjectId = "###{projectId}###" - templateKeyHostName = "###{hostName}###" - templateKeyHostIp = "###{hostIp}###" - templateBuildType = "###{buildType}###" + eBusId = "Collect" ) -func DoAgentCollect() { +func Collect() { + logs.Debug("do Collect") + ipChan := config.EBus.Subscribe(config.IpEvent, eBusId, 1) + defer func() { if err := recover(); err != nil { logs.Error("agent collect panic: ", err) } + config.EBus.Unsubscribe(config.IpEvent, eBusId) }() + for { + ctx, cancel := context.WithCancel(context.Background()) + go func() { + ipData := <-ipChan.DChan + logs.Infof("collect ip change data: %s", ipData.Data) + cancel() + }() + doAgentCollect(ctx) + } +} + +func doAgentCollect(ctx context.Context) { if config.GAgentConfig.CollectorOn == false { logs.Info("agent collector off") return } - if err := writeTelegrafConfig(); err != nil { - logs.WithError(err).Error("writeTelegrafConfig error") + configContent, err := genTelegrafConfig() + if err != nil { + logs.WithError(err).Error("genTelegrafConfig error") return } + logs.Debug("generate telegraf config") + // 每次重启agent要清理掉无意义的telegraf.log日志，重新记录 logFile := fmt.Sprintf("%s/logs/telegraf.log", systemutil.GetWorkDir()) if fileutil.Exists(logFile) { _ = fileutil.TryRemoveFile(logFile) } - tAgent, err := getTelegrafAgent( - fmt.Sprintf("%s/%s", systemutil.GetWorkDir(), telegrafConfigFile), - logFile, - ) + + tAgent, err := getTelegrafAgent(configContent.Bytes(), logFile) if err != nil { - logs.Errorf("init telegraf agent failed: %v", err) + logs.WithError(err).Error("init telegraf agent failed") return } for { logs.Info("launch telegraf agent") - if err = tAgent.Run(context.Background()); err != nil { - logs.Errorf("telegraf agent exit: %v", err) + err = tAgent.Run(ctx) + select { + case <-ctx.Done(): + // 上下文被取消需要返回调用方重新获取上下文，不然一直是取消状态 + logs.Info("telegraf agent ctx done") + return + default: + // 普通的 telegraf 退出直接重新启动即可 + if err != nil { + logs.WithError(err).Error("telegraf agent exit") + } } time.Sleep(telegrafRelaunchTime) } } -func getTelegrafAgent(configFile, logFile string) (*agent.Agent, error) { +func getTelegrafAgent(configData []byte, logFile string) (*agent.Agent, error) { // get a new config and parse configuration from file. c := telegrafConfig.NewConfig() - if err := c.LoadConfig(configFile); err != nil { + if err := c.LoadConfigData(configData); err != nil { return nil, err } @@ -120,52 +135,52 @@ func getTelegrafAgent(configFile, logFile string) (*agent.Agent, error) { RotationMaxArchives: -1, } - logger.SetupLogging(logConfig) + if err := logger.SetupLogging(logConfig); err != nil { + return nil, err + } return agent.NewAgent(c) } -func writeTelegrafConfig() error { +func genTelegrafConfig() (*bytes.Buffer, error) { // 区分 stream 项目使用模板分割，PAC 上线后删除 projectType := "ci" if strings.HasPrefix(config.GAgentConfig.ProjectId, "git_") { projectType = "stream" } - var content bytes.Buffer - tmpl, err := template.New("tmpl").Parse(telegrafconf.TelegrafConf) - if err != nil { - return errors.Wrap(err, "parse telegraf config template err") - } - err = tmpl.Execute(&content, projectType) - if err != nil { - return errors.Wrap(err, "execute telegraf config template err") - } - - configContent := strings.Replace(content.String(), templateKeyAgentId, config.GAgentConfig.AgentId, 2) - configContent = strings.Replace(configContent, templateKeyAgentSecret, config.GAgentConfig.SecretKey, 2) - configContent = strings.Replace(configContent, templateKeyGateway, buildGateway(config.GAgentConfig.Gateway), 1) - configContent = strings.Replace(configContent, templateKeyProjectId, config.GAgentConfig.ProjectId, 2) - configContent = strings.Replace(configContent, templateKeyHostName, config.GAgentEnv.HostName, 1) - configContent = strings.Replace(configContent, templateKeyHostIp, config.GAgentEnv.AgentIp, 1) - configContent = strings.Replace(configContent, templateBuildType, config.GAgentConfig.BuildType, 1) + tlsCa := "" if config.UseCert { - configContent = strings.Replace(configContent, templateKeyTlsCa, `tls_ca = ".cert"`, 1) - } else { - configContent = strings.Replace(configContent, templateKeyTlsCa, "", 1) + tlsCa = `tls_ca = ".cert"` } - err = os.WriteFile(systemutil.GetWorkDir()+"/telegraf.conf", []byte(configContent), 0666) - if err != nil { - return errors.Wrap(err, "write telegraf config err") + buildGateway := config.GAgentConfig.Gateway + if !strings.HasPrefix(buildGateway, "http") { + buildGateway = "http://" + buildGateway } - return nil -} + ip := config.GAgentEnv.GetAgentIp() + templateData := map[string]string{ + "ProjectType": projectType, + "AgentId": config.GAgentConfig.AgentId, + "AgentSecret": config.GAgentConfig.SecretKey, + "Gateway": buildGateway, + "ProjectId": config.GAgentConfig.ProjectId, + "HostName": config.GAgentEnv.HostName, + "HostIp": config.GAgentEnv.GetAgentIp(), + "BuildType": config.GAgentConfig.BuildType, + "TlsCa": tlsCa, + } + logs.Debugf("telegraf agentip %s", ip) -func buildGateway(gateway string) string { - if strings.HasPrefix(gateway, "http") { - return gateway - } else { - return "http://" + gateway + var content = new(bytes.Buffer) + tmpl, err := template.New("tmpl").Parse(telegrafconf.TelegrafConf) + if err != nil { + return nil, errors.Wrap(err, "parse telegraf config template err") } + err = tmpl.Execute(content, templateData) + if err != nil { + return nil, errors.Wrap(err, "execute telegraf config template err") + } + + return content, nil } diff --git a/src/agent/agent/src/pkg/collector/telegrafConf/telegrafConf.go b/src/agent/agent/src/pkg/collector/telegrafConf/telegrafConf.go index 1b22dea730b..cb2196defc8 100644 --- a/src/agent/agent/src/pkg/collector/telegrafConf/telegrafConf.go +++ b/src/agent/agent/src/pkg/collector/telegrafConf/telegrafConf.go @@ -32,11 +32,11 @@ package telegrafconf const TelegrafConf = ` [global_tags] - projectId = "###{projectId}###" - agentId = "###{agentId}###" - agentSecret = "###{agentSecret}###" - hostName = "###{hostName}###" - hostIp = "###{hostIp}###" + projectId = "{{.ProjectId}}" + agentId = "{{.AgentId}}" + agentSecret = "{{.AgentSecret}}" + hostName = "{{.HostName}}" + hostIp = "{{.HostIp}}" [agent] interval = "1m" round_interval = true @@ -51,12 +51,12 @@ const TelegrafConf = ` logfile = "" hostname = "" omit_hostname = false -{{ if eq . "stream" }} +{{ if eq .ProjectType "stream" }} [[outputs.influxdb]] - urls = ["###{gateway}###/ms/environment/api/buildAgent/agent/thirdPartyAgent/agents/metrix"] + urls = ["{{.Gateway}}/ms/environment/api/buildAgent/agent/thirdPartyAgent/agents/metrix"] database = "agentMetrix" skip_database_creation = true - ###{tls_ca}### + {{.TlsCa}} [[inputs.cpu]] percpu = true @@ -76,16 +76,16 @@ const TelegrafConf = ` {{ else }} [[outputs.http]] - url = "###{gateway}###/ms/environment/api/buildAgent/agent/thirdPartyAgent/agents/metrics" + url = "{{.Gateway}}/ms/environment/api/buildAgent/agent/thirdPartyAgent/agents/metrics" # timeout = "5s" method = "POST" data_format = "json" [outputs.http.headers] Content-Type = "application/json; charset=utf-8" - X-DEVOPS-BUILD-TYPE = "###{buildType}###" - X-DEVOPS-PROJECT-ID = "###{projectId}###" - X-DEVOPS-AGENT-ID = "###{agentId}###" - X-DEVOPS-AGENT-SECRET-KEY = "###{agentSecret}###" + X-DEVOPS-BUILD-TYPE = "{{.BuildType}}" + X-DEVOPS-PROJECT-ID = "{{.ProjectId}}" + X-DEVOPS-AGENT-ID = "{{.AgentId}}" + X-DEVOPS-AGENT-SECRET-KEY = "{{.AgentSecret}}" [[inputs.cpu]] percpu = true diff --git a/src/agent/agent/src/pkg/collector/telegrafConf/telegrafConf_out.go b/src/agent/agent/src/pkg/collector/telegrafConf/telegrafConf_out.go index 4720d05d8e8..4fed292a619 100644 --- a/src/agent/agent/src/pkg/collector/telegrafConf/telegrafConf_out.go +++ b/src/agent/agent/src/pkg/collector/telegrafConf/telegrafConf_out.go @@ -32,9 +32,9 @@ package telegrafconf const TelegrafConf = ` [global_tags] - projectId = "###{projectId}###" - agentId = "###{agentId}###" - agentSecret = "###{agentSecret}###" + projectId = "{{.ProjectId}}" + agentId = "{{.AgentId}}" + agentSecret = "{{.AgentSecret}}" [agent] interval = "1m" round_interval = true @@ -50,10 +50,10 @@ const TelegrafConf = ` hostname = "" omit_hostname = false [[outputs.influxdb]] - urls = ["###{gateway}###/ms/environment/api/buildAgent/agent/thirdPartyAgent/agents/metrix"] + urls = ["{{.Gateway}}/ms/environment/api/buildAgent/agent/thirdPartyAgent/agents/metrix"] database = "agentMetrix" skip_database_creation = true - ###{tls_ca}### + {{.TlsCa}} [[inputs.cpu]] percpu = true totalcpu = true diff --git a/src/agent/agent/src/pkg/collector/telegrafConf/telegrafConf_out_win.go b/src/agent/agent/src/pkg/collector/telegrafConf/telegrafConf_out_win.go index 44cd08c47d3..22a180d9aaf 100644 --- a/src/agent/agent/src/pkg/collector/telegrafConf/telegrafConf_out_win.go +++ b/src/agent/agent/src/pkg/collector/telegrafConf/telegrafConf_out_win.go @@ -32,9 +32,9 @@ package telegrafconf const TelegrafConf = ` [global_tags] - projectId = "###{projectId}###" - agentId = "###{agentId}###" - agentSecret = "###{agentSecret}###" + projectId = "{{.ProjectId}}" + agentId = "{{.AgentId}}" + agentSecret = "{{.AgentSecret}}" [agent] interval = "1m" round_interval = true @@ -50,10 +50,10 @@ const TelegrafConf = ` hostname = "" omit_hostname = false [[outputs.influxdb]] - urls = ["###{gateway}###/ms/environment/api/buildAgent/agent/thirdPartyAgent/agents/metrix"] + urls = ["{{.Gateway}}/ms/environment/api/buildAgent/agent/thirdPartyAgent/agents/metrix"] database = "agentMetrix" skip_database_creation = true - ###{tls_ca}### + {{.TlsCa}} [[inputs.mem]] [[inputs.disk]] ignore_fs = ["tmpfs", "devtmpfs", "devfs", "overlay", "aufs", "squashfs"] diff --git a/src/agent/agent/src/pkg/collector/telegrafConf/telegrafConf_win.go b/src/agent/agent/src/pkg/collector/telegrafConf/telegrafConf_win.go index d24b36672c1..187c9a7eff0 100644 --- a/src/agent/agent/src/pkg/collector/telegrafConf/telegrafConf_win.go +++ b/src/agent/agent/src/pkg/collector/telegrafConf/telegrafConf_win.go @@ -32,11 +32,11 @@ package telegrafconf const TelegrafConf = ` [global_tags] - projectId = "###{projectId}###" - agentId = "###{agentId}###" - agentSecret = "###{agentSecret}###" - hostName = "###{hostName}###" - hostIp = "###{hostIp}###" + projectId = "{{.ProjectId}}" + agentId = "{{.AgentId}}" + agentSecret = "{{.AgentSecret}}" + hostName = "{{.HostName}}" + hostIp = "{{.HostIp}}" [agent] interval = "1m" round_interval = true @@ -51,12 +51,12 @@ const TelegrafConf = ` logfile = "" hostname = "" omit_hostname = false -{{ if eq . "stream" }} +{{ if eq .ProjectType "stream" }} [[outputs.influxdb]] - urls = ["###{gateway}###/ms/environment/api/buildAgent/agent/thirdPartyAgent/agents/metrix"] + urls = ["{{.Gateway}}/ms/environment/api/buildAgent/agent/thirdPartyAgent/agents/metrix"] database = "agentMetrix" skip_database_creation = true - ###{tls_ca}### + {{.TlsCa}} [[inputs.mem]] [[inputs.disk]] @@ -154,16 +154,16 @@ const TelegrafConf = ` {{ else }} [[outputs.http]] - url = "###{gateway}###/ms/environment/api/buildAgent/agent/thirdPartyAgent/agents/metrics" + url = "{{.Gateway}}/ms/environment/api/buildAgent/agent/thirdPartyAgent/agents/metrics" # timeout = "5s" method = "POST" data_format = "json" [outputs.http.headers] Content-Type = "application/json; charset=utf-8" - X-DEVOPS-BUILD-TYPE = "###{buildType}###" - X-DEVOPS-PROJECT-ID = "###{projectId}###" - X-DEVOPS-AGENT-ID = "###{agentId}###" - X-DEVOPS-AGENT-SECRET-KEY = "###{agentSecret}###" + X-DEVOPS-BUILD-TYPE = "{{.BuildType}}" + X-DEVOPS-PROJECT-ID = "{{.ProjectId}}" + X-DEVOPS-AGENT-ID = "{{.AgentId}}" + X-DEVOPS-AGENT-SECRET-KEY = "{{.AgentSecret}}" [[inputs.win_perf_counters]] [[inputs.win_perf_counters.object]] diff --git a/src/agent/agent/src/pkg/config/bus.go b/src/agent/agent/src/pkg/config/bus.go new file mode 100644 index 00000000000..60b781d31b9 --- /dev/null +++ b/src/agent/agent/src/pkg/config/bus.go @@ -0,0 +1,92 @@ +package config + +import ( + "github.com/TencentBlueKing/bk-ci/agentcommon/logs" + "sync" +) + +type DataEvent struct { + Data any + Topic DataEventType +} + +type DataEventType string + +const ( + IpEvent DataEventType = "IP" +) + +// DataChannel 是一个能接收 DataEvent 的 channel +type DataChannel struct { + Id string + DChan chan DataEvent +} + +// EventBus 存储有关订阅者感兴趣的特定主题的信息 +type EventBus struct { + subscribers map[DataEventType][]DataChannel + rwLock sync.RWMutex +} + +func (eb *EventBus) Publish(topic DataEventType, data any) { + go func() { + eb.rwLock.RLock() + logs.Infof("EventBus Publish %s %v", topic, data) + defer eb.rwLock.RUnlock() + + if chs, found := eb.subscribers[topic]; found { + for _, ch := range chs { + select { + case ch.DChan <- DataEvent{Data: data, Topic: topic}: + logs.Debugf("EventBus Publish send %s %v", topic, data) + default: + // 管道满了需要丢弃后写入不然会阻塞 + <-ch.DChan + ch.DChan <- DataEvent{Data: data, Topic: topic} + } + + } + } + logs.Debugf("EventBus Publish send over %s %v", topic, data) + }() +} + +func (eb *EventBus) Subscribe(topic DataEventType, id string, chanBuffer int) DataChannel { + ch := DataChannel{ + Id: id, + DChan: make(chan DataEvent, chanBuffer), + } + eb.rwLock.Lock() + logs.Infof("EventBus Subscribe %s %s", topic, ch.Id) + defer eb.rwLock.Unlock() + if prev, found := eb.subscribers[topic]; found { + eb.subscribers[topic] = append(prev, ch) + } else { + eb.subscribers[topic] = append([]DataChannel{}, ch) + } + + return ch +} + +func (eb *EventBus) Unsubscribe(topic DataEventType, id string) { + eb.rwLock.Lock() + logs.Infof("EventBus Unsubscribe %s %s", topic, id) + defer eb.rwLock.Unlock() + slice := eb.subscribers[topic] + for index, ch := range slice { + if ch.Id == id { + logs.Debugf("EventBus Unsubscribe close %s %s", topic, ch.Id) + close(ch.DChan) + eb.subscribers[topic] = append(slice[:index], slice[index+1:]...) + break + } + } +} + +var EBus *EventBus + +func init() { + EBus = &EventBus{ + subscribers: map[DataEventType][]DataChannel{}, + } +} diff --git a/src/agent/agent/src/pkg/config/bus_test.go b/src/agent/agent/src/pkg/config/bus_test.go new file mode 100644 index 00000000000..a84b5ed2cbd --- /dev/null +++ b/src/agent/agent/src/pkg/config/bus_test.go @@ -0,0 +1,50 @@ +package config + +import ( + "bytes" + "github.com/TencentBlueKing/bk-ci/agentcommon/logs" + "strconv" + "sync" + "testing" + "time" +) + +func TestEventBus(t *testing.T) { + logs.UNTestDebugInit() + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + time.Sleep(3 * time.Second) + ipChan := EBus.Subscribe(IpEvent, "go1", 1) + + defer func() { + if err := recover(); err != nil { + logs.Error("agent collect panic: ", err) + } + EBus.Unsubscribe(IpEvent, "go1") + wg.Done() + }() + + data := <-ipChan.DChan + if data.Data != "127.0.0.1" { + t.Errorf("Subscribe error data is %s", data.Data) + } + time.Sleep(3 * time.Second) + + var b *bytes.Buffer + b.Bytes() + }() + + time.Sleep(1 * time.Second) + EBus.Publish(IpEvent, "127.0.0.2") + time.Sleep(3 * time.Second) + EBus.Publish(IpEvent, "127.0.0.1") + for i := 3; i <= 8; i++ { + time.Sleep(1 * time.Second) + EBus.Publish(IpEvent, "127.0.0."+strconv.Itoa(i)) + } + wg.Wait() + if len(EBus.subscribers[IpEvent]) > 0 { + t.Error("unsubscribe error") + } +} diff --git a/src/agent/agent/src/pkg/config/config.go b/src/agent/agent/src/pkg/config/config.go index b054d43d36d..ee6ba743c94 100644 --- a/src/agent/agent/src/pkg/config/config.go +++ b/src/agent/agent/src/pkg/config/config.go @@ -31,8 +31,8 @@ import ( "bytes" "crypto/tls" "crypto/x509" - "errors" "fmt" + "github.com/pkg/errors" "net/http" "os" "path/filepath" @@ -105,30 +105,45 @@ type AgentConfig struct { // AgentEnv Agent 环境配置 type AgentEnv struct { OsName string - AgentIp string + agentIp string HostName string SlaveVersion string AgentVersion string AgentInstallPath string } +func (e *AgentEnv) GetAgentIp() string { + return e.agentIp +} + +func (e *AgentEnv) SetAgentIp(ip string) { + // IP变更时发送事件 + if e.agentIp != "" && e.agentIp != ip && ip != "127.0.0.1" { + EBus.Publish(IpEvent, ip) + } + e.agentIp = ip +} + var GAgentEnv *AgentEnv var GAgentConfig *AgentConfig -var GEnvVars map[string]string var UseCert bool - -var IsDebug bool = false +var IsDebug = false // Init 加载和初始化配置 func Init(isDebug bool) { IsDebug = isDebug err := LoadAgentConfig() if err != nil { - logs.Error("load agent config err: ", err) + logs.WithError(err).Error("load agent config err") systemutil.ExitProcess(1) } initCert() LoadAgentEnv() + + GApiEnvVars = &GEnvVarsT{ + envs: make(map[string]string), + lock: sync.RWMutex{}, + } } // LoadAgentEnv 加载Agent环境 @@ -139,12 +154,11 @@ func LoadAgentEnv() { 忽略一些在Windows机器上VPN代理软件所产生的虚拟网卡（有Mac地址）的IP，一般这类IP 更像是一些路由器的192开头的IP，属于干扰IP，安装了这类软件的windows机器IP都会变成相同，所以需要忽略掉 */ + var splitIps []string if len(GAgentConfig.IgnoreLocalIps) > 0 { - splitIps := util.SplitAndTrimSpace(GAgentConfig.IgnoreLocalIps, ",") - GAgentEnv.AgentIp = systemutil.GetAgentIp(splitIps) - } else { - GAgentEnv.AgentIp = systemutil.GetAgentIp([]string{}) + splitIps = util.SplitAndTrimSpace(GAgentConfig.IgnoreLocalIps, ",") } + GAgentEnv.SetAgentIp(systemutil.GetAgentIp(splitIps)) GAgentEnv.HostName = systemutil.GetHostName() GAgentEnv.OsName = systemutil.GetOsName() @@ -168,7 +182,7 @@ func DetectAgentVersionByDir(workDir string) string { } err := fileutil.SetExecutable(agentExecutable) if err != nil { - logs.Warn(fmt.Errorf("chmod agent file failed: %v", err)) + logs.WithError(err).Warn("chmod agent file failed") return "" } } diff --git a/src/agent/agent/src/pkg/config/env.go b/src/agent/agent/src/pkg/config/env.go new file mode 100644 index 00000000000..494071c552e --- /dev/null +++ b/src/agent/agent/src/pkg/config/env.go @@ -0,0 +1,75 @@ +package config + +import ( + "os" + "strings" + "sync" +) + +// GApiEnvVars 来自页面配置的环境变量 +var GApiEnvVars *GEnvVarsT + +type GEnvVarsT struct { + envs map[string]string + lock sync.RWMutex +} + +func (e *GEnvVarsT) Get(key string) (string, bool) { + e.lock.RLock() + defer e.lock.RUnlock() + res, ok := e.envs[key] + return res, ok +} + +func (e *GEnvVarsT) SetEnvs(envs map[string]string) { + e.lock.Lock() + defer e.lock.Unlock() + e.envs = envs +} + +func (e *GEnvVarsT) RangeDo(do func(k, v string) bool) { + e.lock.RLock() + defer e.lock.RUnlock() + for k, v := range e.envs { + ok := do(k, v) + if !ok { + return + } + } +} + +func (e *GEnvVarsT) Size() int { + e.lock.RLock() + defer e.lock.RUnlock() + return len(e.envs) +} + +// FetchEnvAndCheck 查询是否有某个环境变量，同时校验是否符合要求 +func FetchEnvAndCheck(key string, checkValue string) bool { + v, ok := FetchEnv(key) + if !ok { + return checkValue == "" + } + return v == checkValue +} + +// FetchEnv 查询是否有某个环境变量，需要同时查询系统和后台变量 +func FetchEnv(key string) (string, bool) { + // 优先使用后台配置的 + v, ok := GApiEnvVars.Get(key) + if ok { + return v, true + } + + for _, envStr := range os.Environ() { + parts := strings.Split(envStr, "=") + if len(parts) < 2 { + continue + } + if parts[0] == key { + return parts[1], true + } + } + + return "", false +} diff --git a/src/agent/agent/src/pkg/constant/constant.go b/src/agent/agent/src/pkg/constant/constant.go index 4e24909a71d..f3f591ec326 100644 --- a/src/agent/agent/src/pkg/constant/constant.go +++ b/src/agent/agent/src/pkg/constant/constant.go @@ -1,6 +1,3 @@ -//go:build !out -// +build !out - /* * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. * @@ -30,9 +27,19 @@ package constant -// 用来放一些常量，可能内外部不一致 +import "os" const ( - DockerDataDir = "/data/landun/workspace" - DAEMON_EXIT_CODE = 88 + DaemonExitCode = 88 + + // DevopsAgentEnableNewConsole 如果设为true 则windows启动进程时使用 newConsole + DevopsAgentEnableNewConsole = "DEVOPS_AGENT_ENABLE_NEW_CONSOLE" + // DevopsAgentEnableExitGroup 启动Agent杀掉构建进程组的兜底逻辑 + DevopsAgentEnableExitGroup = "DEVOPS_AGENT_ENABLE_EXIT_GROUP" + + // CommonFileModePerm 公共文件权限 + CommonFileModePerm os.FileMode = 0644 + + // WinCommandNewConsole windwos启动进程时打开新的console窗口 + WinCommandNewConsole = 0x00000010 ) diff --git a/src/agent/agent/src/pkg/constant/constant_inner.go b/src/agent/agent/src/pkg/constant/constant_inner.go new file mode 100644 index 00000000000..3ae322273dc --- /dev/null +++ b/src/agent/agent/src/pkg/constant/constant_inner.go @@ -0,0 +1,37 @@ +//go:build !out +// +build !out + +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package constant + +// 用来放一些常量，可能内外部不一致 + +const ( + DockerDataDir = "/data/landun/workspace" +) diff --git a/src/agent/agent/src/pkg/constant/constant_out.go b/src/agent/agent/src/pkg/constant/constant_out.go index 412d636e906..2600a5b10bf 100644 --- a/src/agent/agent/src/pkg/constant/constant_out.go +++ b/src/agent/agent/src/pkg/constant/constant_out.go @@ -33,6 +33,5 @@ package constant // 用来放一些常量，可能内外部不一致 const ( - DockerDataDir = "/data/devops/workspace" - DAEMON_EXIT_CODE = 88 + DockerDataDir = "/data/devops/workspace" ) diff --git a/src/agent/agent/src/pkg/cron/cron.go b/src/agent/agent/src/pkg/cron/cron.go index 0ab74ab260b..c904810b3a4 100644 --- a/src/agent/agent/src/pkg/cron/cron.go +++ b/src/agent/agent/src/pkg/cron/cron.go @@ -145,7 +145,7 @@ func cleanLogFile(timeBeforeInHours int) { } dockerFiles, err := os.ReadDir(dockerLogDir) if err != nil { - logs.Error("read docker log dir error: ", err.Error()) + logs.WithError(err).Error("read docker log dir error") return } diff --git a/src/agent/agent/src/pkg/exiterror/exiterror.go b/src/agent/agent/src/pkg/exiterror/exiterror.go index f2a0251bead..d7ffa49df37 100644 --- a/src/agent/agent/src/pkg/exiterror/exiterror.go +++ b/src/agent/agent/src/pkg/exiterror/exiterror.go @@ -1,8 +1,8 @@ package exitcode import ( - "errors" "fmt" + "github.com/pkg/errors" "os" "strings" "sync/atomic" @@ -47,7 +47,7 @@ func Exit(exitError *ExitErrorType) { if exitError != nil { logs.Errorf("ExitError|%s|%s", exitError.ErrorEnum, exitError.Message) } - os.Exit(constant.DAEMON_EXIT_CODE) + os.Exit(constant.DaemonExitCode) } func WriteFileWithCheck(name string, data []byte, perm os.FileMode) error { @@ -80,7 +80,7 @@ func CheckOsIoError(path string, err error) { } // 避免不必要的错杀，信号错误最少连续持续 10 次以上才能杀掉 -var jdkSignFlag atomic.Int32 = atomic.Int32{} +var jdkSignFlag = atomic.Int32{} func CheckSignalJdkError(err error) { if err == nil { @@ -110,7 +110,7 @@ func CheckSignalJdkError(err error) { } } -var workerSignFlag atomic.Int32 = atomic.Int32{} +var workerSignFlag = atomic.Int32{} func CheckSignalWorkerError(err error) { if err == nil { diff --git a/src/agent/agent/src/pkg/imagedebug/imagedebug.go b/src/agent/agent/src/pkg/imagedebug/imagedebug.go index 4d61bcc552d..42241c4dfbc 100644 --- a/src/agent/agent/src/pkg/imagedebug/imagedebug.go +++ b/src/agent/agent/src/pkg/imagedebug/imagedebug.go @@ -227,7 +227,7 @@ func CreateDebugContainer( RegistryAuth: auth, }) if err != nil { - imageDebugLogs.Errorf("pull new image %s error %s", imageName, err.Error()) + imageDebugLogs.WithError(err).Errorf("pull new image %s error", imageName) return errors.New(i18n.Localize("PullImageError", map[string]interface{}{"name": imageName, "err": err.Error()})) } defer reader.Close() @@ -245,7 +245,7 @@ func CreateDebugContainer( } // 解析docker options - dockerConfig, err := job_docker.ParseDockeroptions(cli, debugInfo.Options) + dockerConfig, err := job_docker.ParseDockerOptions(cli, debugInfo.Options) if err != nil { imageDebugLogs.Error(err.Error()) return err @@ -515,7 +515,7 @@ func CreateExecServer( Cmd: conf.Cmd, }) - url := fmt.Sprintf("ws://%s:%d/start_exec?exec_id=%s&container_id=%s", config.GAgentEnv.AgentIp, conf.Port, exec.ID, containerId) + url := fmt.Sprintf("ws://%s:%d/start_exec?exec_id=%s&container_id=%s", config.GAgentEnv.GetAgentIp(), conf.Port, exec.ID, containerId) // 上报结束并附带 url imageDebugLogs.Infof("ws url: %s", url) diff --git a/src/agent/agent/src/pkg/imagedebug/manager.go b/src/agent/agent/src/pkg/imagedebug/manager.go index ab5a4e83ca7..db095a4c852 100644 --- a/src/agent/agent/src/pkg/imagedebug/manager.go +++ b/src/agent/agent/src/pkg/imagedebug/manager.go @@ -30,7 +30,7 @@ type Manager interface { Start() error CreateExecNoHttp(*WebSocketConfig) (*docker.Exec, error) - // handler container web console + // StartExec handler container web console StartExec(http.ResponseWriter, *http.Request, *WebSocketConfig) CreateExec(http.ResponseWriter, *http.Request, *WebSocketConfig) ResizeExec(http.ResponseWriter, *http.Request, *WebSocketConfig) diff --git a/src/agent/agent/src/pkg/installer/installer.go b/src/agent/agent/src/pkg/installer/installer.go index d27b0e58803..f3a069c8a35 100644 --- a/src/agent/agent/src/pkg/installer/installer.go +++ b/src/agent/agent/src/pkg/installer/installer.go @@ -28,8 +28,8 @@ package installer import ( - "errors" "fmt" + "github.com/pkg/errors" "github.com/TencentBlueKing/bk-ci/agentcommon/logs" @@ -53,7 +53,7 @@ func DoInstallAgent() error { totalLock := flock.New(fmt.Sprintf("%s/%s.lock", systemutil.GetRuntimeDir(), systemutil.TotalLock)) err := totalLock.Lock() if err = totalLock.Lock(); err != nil { - logs.Error("get total lock failed, exit", err.Error()) + logs.WithError(err).Error("get total lock failed, exit") return errors.New("get total lock failed") } defer func() { totalLock.Unlock() }() @@ -125,7 +125,7 @@ func InstallAgent() error { err := fileutil.SetExecutable(startCmd) if err != nil { - return fmt.Errorf("chmod install script failed: %s", err.Error()) + return errors.Wrap(err, "chmod install script failed") } output, err := command.RunCommand(startCmd, []string{} /*args*/, workDir, nil) diff --git a/src/agent/agent/src/pkg/job/build.go b/src/agent/agent/src/pkg/job/build.go index 31c33a32fb2..b632ef86daf 100644 --- a/src/agent/agent/src/pkg/job/build.go +++ b/src/agent/agent/src/pkg/job/build.go @@ -30,8 +30,8 @@ package job import ( "encoding/base64" "encoding/json" - "errors" "fmt" + "github.com/pkg/errors" "io/fs" "os" "strings" @@ -44,7 +44,6 @@ import ( "github.com/TencentBlueKing/bk-ci/agent/src/pkg/config" exitcode "github.com/TencentBlueKing/bk-ci/agent/src/pkg/exiterror" "github.com/TencentBlueKing/bk-ci/agent/src/pkg/i18n" - "github.com/TencentBlueKing/bk-ci/agent/src/pkg/util/command" "github.com/TencentBlueKing/bk-ci/agent/src/pkg/util/httputil" "github.com/TencentBlueKing/bk-ci/agent/src/pkg/util/systemutil" "github.com/TencentBlueKing/bk-ci/agentcommon/utils/fileutil" @@ -61,8 +60,6 @@ func init() { BuildTotalManager = new(BuildTotalManagerType) } -const buildIntervalInSeconds = 5 - // AgentStartup 上报构建机启动 func AgentStartup() (agentStatus string, err error) { result, err := api.AgentStartup() @@ -72,7 +69,7 @@ func AgentStartup() (agentStatus string, err error) { // parseAgentStatusResult 解析状态信息 func parseAgentStatusResult(result *httputil.DevopsResult, resultErr error) (agentStatus string, err error) { if resultErr != nil { - logs.Error("parse agent status error: ", resultErr.Error()) + logs.WithErrorNoStack(resultErr).Error("parse agent status error") return "", errors.New("parse agent status error") } if result.IsNotOk() { @@ -131,11 +128,11 @@ func DoBuild(buildInfo *api.ThirdPartyBuildInfo) { err := runBuild(buildInfo) if err != nil { - logs.Error("start build failed: ", err.Error()) + logs.WithError(err).Error("start build failed") } } -// checkParallelTaskCount 检查当前运行的最大任务数 +// CheckParallelTaskCount checkParallelTaskCount 检查当前运行的最大任务数 func CheckParallelTaskCount() (dockerCanRun bool, normalCanRun bool) { // 检查docker任务 dockerInstanceCount := GBuildDockerManager.GetInstanceCount() @@ -211,10 +208,11 @@ func runBuild(buildInfo *api.ThirdPartyBuildInfo) error { "DEVOPS_GATEWAY": config.GetGateWay(), "BK_CI_LOCALE_LANGUAGE": config.GAgentConfig.Language, } - if config.GEnvVars != nil { - for k, v := range config.GEnvVars { + if config.GApiEnvVars != nil { + config.GApiEnvVars.RangeDo(func(k, v string) bool { goEnv[k] = v - } + return true + }) } // #5806 定义临时目录 tmpDir, tmpMkErr := systemutil.MkBuildTmpDir() @@ -224,50 +222,11 @@ func runBuild(buildInfo *api.ThirdPartyBuildInfo) error { workerBuildFinish(buildInfo.ToFinish(false, errMsg, api.MakeTmpDirErrorEnum)) return tmpMkErr } - if systemutil.IsWindows() { - startCmd := config.GetJava() - agentLogPrefix := fmt.Sprintf("%s_%s_agent", buildInfo.BuildId, buildInfo.VmSeqId) - errorMsgFile := getWorkerErrorMsgFile(buildInfo.BuildId, buildInfo.VmSeqId) - args := []string{ - "-Djava.io.tmpdir=" + tmpDir, - "-Ddevops.agent.error.file=" + errorMsgFile, - "-Dbuild.type=AGENT", - "-DAGENT_LOG_PREFIX=" + agentLogPrefix, - "-Xmx2g", // #5806 兼容性问题，必须独立一行 - "-jar", - config.BuildAgentJarPath(), - getEncodedBuildInfo(buildInfo)} - pid, err := command.StartProcess(startCmd, args, workDir, goEnv, runUser) - if err != nil { - errMsg := i18n.Localize("StartWorkerProcessFailed", map[string]interface{}{"err": err.Error()}) - logs.Error(errMsg) - workerBuildFinish(buildInfo.ToFinish(false, errMsg, api.BuildProcessStartErrorEnum)) - return err - } - // 添加需要构建结束后删除的文件 - buildInfo.ToDelTmpFiles = []string{errorMsgFile} - GBuildManager.AddBuild(pid, buildInfo) - logs.Info(fmt.Sprintf("[%s]|Job#_%s|Build started, pid:%d ", buildInfo.BuildId, buildInfo.VmSeqId, pid)) - return nil - } else { - startScriptFile, err := writeStartBuildAgentScript(buildInfo, tmpDir) - if err != nil { - errMsg := i18n.Localize("CreateStartScriptFailed", map[string]interface{}{"err": err.Error()}) - logs.Error(errMsg) - workerBuildFinish(buildInfo.ToFinish(false, errMsg, api.PrepareScriptCreateErrorEnum)) - return err - } - pid, err := command.StartProcess(startScriptFile, []string{}, workDir, goEnv, runUser) - if err != nil { - errMsg := i18n.Localize("StartWorkerProcessFailed", map[string]interface{}{"err": err.Error()}) - logs.Error(errMsg) - workerBuildFinish(buildInfo.ToFinish(false, errMsg, api.BuildProcessStartErrorEnum)) - return err - } - GBuildManager.AddBuild(pid, buildInfo) - logs.Info(fmt.Sprintf("[%s]|Job#_%s|Build started, pid:%d ", buildInfo.BuildId, buildInfo.VmSeqId, pid)) + if err := doBuild(buildInfo, tmpDir, workDir, goEnv, runUser); err != nil { + return err } + return nil } @@ -279,71 +238,6 @@ func getEncodedBuildInfo(buildInfo *api.ThirdPartyBuildInfo) string { return codedBuildInfo } -func writeStartBuildAgentScript(buildInfo *api.ThirdPartyBuildInfo, tmpDir string) (string, error) { - logs.Info("write start build agent script to file") - // 套娃，多加一层脚本，使用exec新起进程，这样才会读取 .bash_profile - prepareScriptFile := fmt.Sprintf( - "%s/devops_agent_prepare_start_%s_%s_%s.sh", - systemutil.GetWorkDir(), buildInfo.ProjectId, buildInfo.BuildId, buildInfo.VmSeqId) - scriptFile := fmt.Sprintf( - "%s/devops_agent_start_%s_%s_%s.sh", - systemutil.GetWorkDir(), buildInfo.ProjectId, buildInfo.BuildId, buildInfo.VmSeqId) - - errorMsgFile := getWorkerErrorMsgFile(buildInfo.BuildId, buildInfo.VmSeqId) - buildInfo.ToDelTmpFiles = []string{ - scriptFile, prepareScriptFile, errorMsgFile, - } - - logs.Info("start agent script: ", scriptFile) - agentLogPrefix := fmt.Sprintf("%s_%s_agent", buildInfo.BuildId, buildInfo.VmSeqId) - lines := []string{ - "#!" + getCurrentShell(), - fmt.Sprintf("cd %s", systemutil.GetWorkDir()), - fmt.Sprintf("%s -Ddevops.slave.agent.start.file=%s -Ddevops.slave.agent.prepare.start.file=%s "+ - "-Ddevops.agent.error.file=%s "+ - "-Dbuild.type=AGENT -DAGENT_LOG_PREFIX=%s -Xmx2g -Djava.io.tmpdir=%s -jar %s %s", - config.GetJava(), scriptFile, prepareScriptFile, - errorMsgFile, - agentLogPrefix, tmpDir, config.BuildAgentJarPath(), getEncodedBuildInfo(buildInfo)), - } - scriptContent := strings.Join(lines, "\n") - - err := exitcode.WriteFileWithCheck(scriptFile, []byte(scriptContent), os.ModePerm) - defer func() { - _ = systemutil.Chmod(scriptFile, os.ModePerm) - _ = systemutil.Chmod(prepareScriptFile, os.ModePerm) - }() - if err != nil { - return "", err - } else { - prepareScriptContent := strings.Join(getShellLines(scriptFile), "\n") - err := exitcode.WriteFileWithCheck(prepareScriptFile, []byte(prepareScriptContent), os.ModePerm) - if err != nil { - return "", err - } else { - return prepareScriptFile, nil - } - } -} - -// getShellLines 根据不同的shell的参数要求，这里可能需要不同的参数或者参数顺序 -func getShellLines(scriptFile string) (newLines []string) { - shell := getCurrentShell() - switch shell { - case "/bin/tcsh": - newLines = []string{ - "#!" + shell, - "exec " + shell + " " + scriptFile + " -l", - } - default: - newLines = []string{ - "#!" + shell, - "exec " + shell + " -l " + scriptFile, - } - } - return newLines -} - func workerBuildFinish(buildInfo *api.ThirdPartyBuildWithStatus) { if buildInfo == nil { logs.Warn("buildInfo not exist") @@ -367,7 +261,7 @@ func workerBuildFinish(buildInfo *api.ThirdPartyBuildWithStatus) { } result, err := api.WorkerBuildFinish(buildInfo) if err != nil { - logs.Error("send worker build finish failed: ", err.Error()) + logs.WithErrorNoStack(err).Error("send worker build finish failed") } if result.IsNotOk() { logs.Error("send worker build finish failed: ", result.Message) @@ -428,15 +322,38 @@ func removeFileThan7Days(dir string, f fs.DirEntry) { } } -func getCurrentShell() (shell string) { - if config.GAgentConfig.DetectShell { - shell = os.Getenv("SHELL") - if strings.TrimSpace(shell) == "" { - shell = "/bin/bash" - } - } else { - shell = "/bin/bash" +const ( + errorMsgFileSuffix = "build_msg.log" + prepareStartScriptFilePrefix = "devops_agent_prepare_start" + prepareStartScriptFileSuffix = ".sh" + startScriptFilePrefix = "devops_agent_start" + startScriptFileSuffix = ".sh" +) + +// getWorkerErrorMsgFile 获取worker执行错误信息的日志文件 +func getWorkerErrorMsgFile(buildId, vmSeqId string) string { + return fmt.Sprintf("%s/build_tmp/%s_%s_%s", + systemutil.GetWorkDir(), buildId, vmSeqId, errorMsgFileSuffix) +} + +// getUnixWorkerPrepareStartScriptFile 获取unix系统，主要是darwin和linux的prepare start script文件 +func getUnixWorkerPrepareStartScriptFile(projectId, buildId, vmSeqId string) string { + return fmt.Sprintf("%s/%s_%s_%s_%s%s", + systemutil.GetWorkDir(), prepareStartScriptFilePrefix, projectId, buildId, vmSeqId, prepareStartScriptFileSuffix) +} + +// getUnixWorkerStartScriptFile 获取unix系统，主要是darwin和linux的prepare start script文件 +func getUnixWorkerStartScriptFile(projectId, buildId, vmSeqId string) string { + return fmt.Sprintf("%s/%s_%s_%s_%s%s", + systemutil.GetWorkDir(), startScriptFilePrefix, projectId, buildId, vmSeqId, startScriptFileSuffix) +} + +// CheckRunningJob 校验当前是否有正在跑的任务 +func CheckRunningJob() bool { + if GBuildManager.GetPreInstancesCount() > 0 || + GBuildManager.GetInstanceCount() > 0 || + GBuildDockerManager.GetInstanceCount() > 0 { + return true } - logs.Info("current shell: ", shell) - return + return false } diff --git a/src/agent/agent/src/pkg/job/build_docker.go b/src/agent/agent/src/pkg/job/build_docker.go index 340c19bbd6f..3be9fc9b91a 100644 --- a/src/agent/agent/src/pkg/job/build_docker.go +++ b/src/agent/agent/src/pkg/job/build_docker.go @@ -30,6 +30,7 @@ package job import ( "context" "fmt" + "github.com/pkg/errors" "io" "os" "path/filepath" @@ -52,7 +53,6 @@ import ( "github.com/docker/docker/api/types/mount" "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" - "github.com/pkg/errors" ) // buildDockerManager docker构建机构建对象管理 @@ -160,7 +160,7 @@ func doDockerJob(buildInfo *api.ThirdPartyBuildInfo) { ctx := context.Background() cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) if err != nil { - logs.Error("DOCKER_JOB|create docker client error ", err) + logs.WithError(err).Error("DOCKER_JOB|create docker client error") dockerBuildFinish(buildInfo.ToFinish(false, i18n.Localize("LinkDockerError", map[string]interface{}{"err": err}), api.DockerClientCreateErrorEnum)) return } @@ -170,7 +170,7 @@ func doDockerJob(buildInfo *api.ThirdPartyBuildInfo) { // 判断本地是否已经有镜像了 images, err := cli.ImageList(ctx, types.ImageListOptions{}) if err != nil { - logs.Error("DOCKER_JOB|list docker images error ", err) + logs.WithError(err).Error("DOCKER_JOB|list docker images error") dockerBuildFinish(buildInfo.ToFinish(false, i18n.Localize("GetDockerImagesError", map[string]interface{}{"err": err}), api.DockerImagesFetchErrorEnum)) return } @@ -212,7 +212,7 @@ func doDockerJob(buildInfo *api.ThirdPartyBuildInfo) { RegistryAuth: auth, }) if err != nil { - logs.Error(fmt.Sprintf("DOCKER_JOB|pull new image %s error ", imageName), err) + logs.WithError(err).Errorf("DOCKER_JOB|pull new image %s error", imageName) dockerBuildFinish(buildInfo.ToFinish(false, i18n.Localize("PullImageError", map[string]interface{}{"name": imageName, "err": err.Error()}), api.DockerImagePullErrorEnum)) return } @@ -220,7 +220,7 @@ func doDockerJob(buildInfo *api.ThirdPartyBuildInfo) { buf := new(strings.Builder) _, err = io.Copy(buf, reader) if err != nil { - logs.Error("DOCKER_JOB|write image message error ", err) + logs.WithError(err).Error("DOCKER_JOB|write image message error") postLog(true, i18n.Localize("GetPullImageLogError", map[string]interface{}{"err": err.Error()}), buildInfo, api.LogtypeLog) } else { // 异步打印，防止过大卡住主流程 @@ -241,9 +241,9 @@ func doDockerJob(buildInfo *api.ThirdPartyBuildInfo) { } // 解析docker options - dockerConfig, err := job_docker.ParseDockeroptions(cli, dockerBuildInfo.Options) + dockerConfig, err := job_docker.ParseDockerOptions(cli, dockerBuildInfo.Options) if err != nil { - logs.Error("DOCKER_JOB|" + err.Error()) + logs.WithError(err).Error("DOCKER_JOB|") dockerBuildFinish(buildInfo.ToFinish(false, err.Error(), api.DockerDockerOptionsErrorEnum)) return } @@ -253,7 +253,7 @@ func doDockerJob(buildInfo *api.ThirdPartyBuildInfo) { mounts, err := parseContainerMounts(buildInfo) if err != nil { errMsg := i18n.Localize("ReadDockerMountsError", map[string]interface{}{"err": err.Error()}) - logs.Error("DOCKER_JOB| ", err) + logs.WithError(err).Error("DOCKER_JOB|") dockerBuildFinish(buildInfo.ToFinish(false, errMsg, api.DockerMountCreateErrorEnum)) return } @@ -293,7 +293,7 @@ func doDockerJob(buildInfo *api.ThirdPartyBuildInfo) { creatResp, err := cli.ContainerCreate(ctx, confg, hostConfig, netConfig, nil, containerName) if err != nil { - logs.Error(fmt.Sprintf("DOCKER_JOB|create container %s error ", containerName), err) + logs.WithError(err).Errorf("DOCKER_JOB|create container %s error", containerName) dockerBuildFinish(buildInfo.ToFinish( false, i18n.Localize("CreateContainerError", map[string]interface{}{"name": containerName, "err": err.Error()}), @@ -308,13 +308,13 @@ func doDockerJob(buildInfo *api.ThirdPartyBuildInfo) { return } if err = cli.ContainerRemove(ctx, creatResp.ID, types.ContainerRemoveOptions{Force: true}); err != nil { - logs.Error(fmt.Sprintf("DOCKER_JOB|remove container %s error ", creatResp.ID), err) + logs.WithError(err).Errorf("DOCKER_JOB|remove container %s error", creatResp.ID) } }() // 启动容器 if err := cli.ContainerStart(ctx, creatResp.ID, types.ContainerStartOptions{}); err != nil { - logs.Error(fmt.Sprintf("DOCKER_JOB|start container %s error ", creatResp.ID), err) + logs.WithError(err).Errorf("DOCKER_JOB|start container %s error", creatResp.ID) dockerBuildFinish(buildInfo.ToFinish( false, i18n.Localize("StartContainerError", map[string]interface{}{"name": containerName, "err": err.Error()}), @@ -328,7 +328,7 @@ func doDockerJob(buildInfo *api.ThirdPartyBuildInfo) { select { case err := <-errCh: if err != nil { - logs.Error(fmt.Sprintf("DOCKER_JOB|wait container %s over error ", creatResp.ID), err) + logs.WithError(err).Errorf("DOCKER_JOB|wait container %s over error ", creatResp.ID) dockerBuildFinish(buildInfo.ToFinish( false, i18n.Localize("WaitContainerError", map[string]interface{}{"name": containerName, "err": err.Error()}), @@ -338,7 +338,7 @@ func doDockerJob(buildInfo *api.ThirdPartyBuildInfo) { } case status := <-statusCh: if status.Error != nil { - logs.Error(fmt.Sprintf("DOCKER_JOB|wait container %s over error ", creatResp.ID), status.Error) + logs.Errorf("DOCKER_JOB|wait container %s over error %v", creatResp.ID, status.Error) dockerBuildFinish(buildInfo.ToFinish( false, i18n.Localize("WaitContainerError", map[string]interface{}{"name": containerName, "err": status.Error.Message}), diff --git a/src/agent/agent/src/pkg/job/build_manager.go b/src/agent/agent/src/pkg/job/build_manager.go index 8b483e7aca3..3d1e78b6314 100644 --- a/src/agent/agent/src/pkg/job/build_manager.go +++ b/src/agent/agent/src/pkg/job/build_manager.go @@ -30,16 +30,12 @@ package job import ( "encoding/json" "fmt" - "os" "strings" "sync" "github.com/TencentBlueKing/bk-ci/agentcommon/logs" "github.com/TencentBlueKing/bk-ci/agent/src/pkg/api" - "github.com/TencentBlueKing/bk-ci/agent/src/pkg/i18n" - "github.com/TencentBlueKing/bk-ci/agent/src/pkg/util/systemutil" - "github.com/TencentBlueKing/bk-ci/agentcommon/utils/fileutil" ) // buildManager 二进制构建对象管理 @@ -90,54 +86,10 @@ func (b *buildManager) AddBuild(processId int, buildInfo *api.ThirdPartyBuildInf b.instances.Store(processId, buildInfo) // 启动构建了就删除preInstance b.DeletePreInstance(buildInfo.BuildId) - - // #5806 预先录入异常信息，在构建进程正常结束时清理掉。如果没清理掉，则说明进程非正常退出，可能被OS或人为杀死 - errorMsgFile := getWorkerErrorMsgFile(buildInfo.BuildId, buildInfo.VmSeqId) - _ = fileutil.WriteString(errorMsgFile, i18n.Localize("BuilderProcessWasKilled", nil)) - _ = systemutil.Chmod(errorMsgFile, os.ModePerm) - b.waitProcessDone(processId) } -func (b *buildManager) waitProcessDone(processId int) { - process, err := os.FindProcess(processId) - inf, ok := b.instances.Load(processId) - var info *api.ThirdPartyBuildInfo - if ok { - info = inf.(*api.ThirdPartyBuildInfo) - } - if err != nil { - errMsg := i18n.Localize("BuildProcessErr", map[string]interface{}{"pid": processId, "err": err.Error()}) - logs.Warn(errMsg) - b.instances.Delete(processId) - workerBuildFinish(info.ToFinish(false, errMsg, api.BuildProcessRunErrorEnum)) - return - } - - state, err := process.Wait() - // #5806 从b-xxxx_build_msg.log 读取错误信息，此信息可由worker-agent.jar写入，用于当异常时能够将信息上报给服务器 - msgFile := getWorkerErrorMsgFile(info.BuildId, info.VmSeqId) - msg, _ := fileutil.GetString(msgFile) - logs.Info(fmt.Sprintf("build[%s] pid[%d] finish, state=%v err=%v, msg=%s", info.BuildId, processId, state, err, msg)) - - if err != nil { - if len(msg) == 0 { - msg = err.Error() - } - } - success := true - if len(msg) == 0 { - msg = i18n.Localize("WorkerExit", map[string]interface{}{"pid": processId}) - } else { - success = false - } - - buildInfo := info +func (b *buildManager) DeleteBuild(processId int) { b.instances.Delete(processId) - if success { - workerBuildFinish(buildInfo.ToFinish(success, msg, api.NoErrorEnum)) - } else { - workerBuildFinish(buildInfo.ToFinish(success, msg, api.BuildProcessRunErrorEnum)) - } } func (b *buildManager) GetPreInstancesCount() int { diff --git a/src/agent/agent/src/pkg/job/job_test.go b/src/agent/agent/src/pkg/job/build_test.go similarity index 100% rename from src/agent/agent/src/pkg/job/job_test.go rename to src/agent/agent/src/pkg/job/build_test.go diff --git a/src/agent/agent/src/pkg/job/do_build.go b/src/agent/agent/src/pkg/job/do_build.go new file mode 100644 index 00000000000..d2cb79afc2f --- /dev/null +++ b/src/agent/agent/src/pkg/job/do_build.go @@ -0,0 +1,269 @@ +//go:build linux || darwin +// +build linux darwin + +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package job + +import ( + "fmt" + "github.com/pkg/errors" + "os" + "os/exec" + "runtime" + "strings" + "syscall" + + "github.com/TencentBlueKing/bk-ci/agent/src/pkg/api" + "github.com/TencentBlueKing/bk-ci/agent/src/pkg/config" + "github.com/TencentBlueKing/bk-ci/agent/src/pkg/constant" + exitcode "github.com/TencentBlueKing/bk-ci/agent/src/pkg/exiterror" + "github.com/TencentBlueKing/bk-ci/agent/src/pkg/i18n" + ucommand "github.com/TencentBlueKing/bk-ci/agent/src/pkg/util/command" + "github.com/TencentBlueKing/bk-ci/agent/src/pkg/util/systemutil" + "github.com/TencentBlueKing/bk-ci/agentcommon/logs" + "github.com/TencentBlueKing/bk-ci/agentcommon/utils/fileutil" +) + +func doBuild( + buildInfo *api.ThirdPartyBuildInfo, + tmpDir string, + workDir string, + goEnv map[string]string, + runUser string, +) error { + startScriptFile, err := writeStartBuildAgentScript(buildInfo, tmpDir) + if err != nil { + errMsg := i18n.Localize("CreateStartScriptFailed", map[string]interface{}{"err": err.Error()}) + logs.Error(errMsg) + workerBuildFinish(buildInfo.ToFinish(false, errMsg, api.PrepareScriptCreateErrorEnum)) + return err + } + + enableExitGroup := config.FetchEnvAndCheck(constant.DevopsAgentEnableExitGroup, "true") || + (systemutil.IsMacos() && runtime.GOARCH == "arm64") + if enableExitGroup { + logs.Infof("%s enable exit group", buildInfo.BuildId) + } + cmd, err := StartProcessCmd(startScriptFile, []string{}, workDir, goEnv, runUser, enableExitGroup) + if err != nil { + errMsg := i18n.Localize("StartWorkerProcessFailed", map[string]interface{}{"err": err.Error()}) + logs.Error(errMsg) + workerBuildFinish(buildInfo.ToFinish(false, errMsg, api.BuildProcessStartErrorEnum)) + return err + } + + pid := cmd.Process.Pid + GBuildManager.AddBuild(pid, buildInfo) + logs.Info(fmt.Sprintf("[%s]|Job#_%s|Build started, pid:%d ", buildInfo.BuildId, buildInfo.VmSeqId, pid)) + + // #5806 预先录入异常信息，在构建进程正常结束时清理掉。如果没清理掉，则说明进程非正常退出，可能被OS或人为杀死 + errorMsgFile := getWorkerErrorMsgFile(buildInfo.BuildId, buildInfo.VmSeqId) + _ = fileutil.WriteString(errorMsgFile, i18n.Localize("BuilderProcessWasKilled", nil)) + _ = systemutil.Chmod(errorMsgFile, os.ModePerm) + + if enableExitGroup { + pgId, errPg := syscall.Getpgid(pid) + if errPg != nil { + logs.Errorf("%s %d get pgid error %s", buildInfo.BuildId, pid, errPg.Error()) + } + err = cmd.Wait() + if errPg == nil { + go func() { + logs.Infof("%s do kill %d process group %d", buildInfo.BuildId, pid, pgId) + // 杀死进程组 + errPg = syscall.Kill(-pgId, syscall.SIGKILL) + if errPg != nil { + logs.Errorf("%s failed to kill %d process group %d : %s", buildInfo.BuildId, pid, pgId, errPg.Error()) + return + } + }() + } + } else { + err = cmd.Wait() + } + // #5806 从b-xxxx_build_msg.log 读取错误信息，此信息可由worker-agent.jar写入，用于当异常时能够将信息上报给服务器 + msgFile := getWorkerErrorMsgFile(buildInfo.BuildId, buildInfo.VmSeqId) + msg, _ := fileutil.GetString(msgFile) + if err != nil { + logs.Errorf("build[%s] pid[%d] finish, state=%v err=%v, msg=%s", buildInfo.BuildId, pid, cmd.ProcessState, err, msg) + } else { + logs.Infof("build[%s] pid[%d] finish, state=%v err=%v, msg=%s", buildInfo.BuildId, pid, cmd.ProcessState, err, msg) + } + + // #10362 Worker杀掉当前进程父进程导致Agent误报 + // agent 改动后可能会导致业务执行完成但是进程被杀掉导致流水线错误，所以将错误只是作为额外信息添加 + cmdErrMsg := "" + if err != nil { + cmdErrMsg = "|" + err.Error() + } + + success := true + if len(msg) == 0 { + msg = i18n.Localize("WorkerExit", map[string]interface{}{"pid": pid}) + cmdErrMsg + } else { + msg += cmdErrMsg + success = false + } + + GBuildManager.DeleteBuild(pid) + if success { + workerBuildFinish(buildInfo.ToFinish(success, msg, api.NoErrorEnum)) + } else { + workerBuildFinish(buildInfo.ToFinish(success, msg, api.BuildProcessRunErrorEnum)) + } + + return nil +} + +func writeStartBuildAgentScript(buildInfo *api.ThirdPartyBuildInfo, tmpDir string) (string, error) { + logs.Info("write start build agent script to file") + // 套娃，多加一层脚本，使用exec新起进程，这样才会读取 .bash_profile + prepareScriptFile := fmt.Sprintf( + "%s/devops_agent_prepare_start_%s_%s_%s.sh", + systemutil.GetWorkDir(), buildInfo.ProjectId, buildInfo.BuildId, buildInfo.VmSeqId) + scriptFile := fmt.Sprintf( + "%s/devops_agent_start_%s_%s_%s.sh", + systemutil.GetWorkDir(), buildInfo.ProjectId, buildInfo.BuildId, buildInfo.VmSeqId) + + errorMsgFile := getWorkerErrorMsgFile(buildInfo.BuildId, buildInfo.VmSeqId) + buildInfo.ToDelTmpFiles = []string{ + scriptFile, prepareScriptFile, errorMsgFile, + } + + logs.Info("start agent script: ", scriptFile) + agentLogPrefix := fmt.Sprintf("%s_%s_agent", buildInfo.BuildId, buildInfo.VmSeqId) + lines := []string{ + "#!" + getCurrentShell(), + fmt.Sprintf("cd %s", systemutil.GetWorkDir()), + fmt.Sprintf("%s -Ddevops.slave.agent.start.file=%s -Ddevops.slave.agent.prepare.start.file=%s "+ + "-Ddevops.agent.error.file=%s "+ + "-Dbuild.type=AGENT -DAGENT_LOG_PREFIX=%s -Xmx2g -Djava.io.tmpdir=%s -jar %s %s", + config.GetJava(), scriptFile, prepareScriptFile, + errorMsgFile, + agentLogPrefix, tmpDir, config.BuildAgentJarPath(), getEncodedBuildInfo(buildInfo)), + } + scriptContent := strings.Join(lines, "\n") + + err := exitcode.WriteFileWithCheck(scriptFile, []byte(scriptContent), os.ModePerm) + defer func() { + _ = systemutil.Chmod(scriptFile, os.ModePerm) + _ = systemutil.Chmod(prepareScriptFile, os.ModePerm) + }() + if err != nil { + return "", err + } else { + prepareScriptContent := strings.Join(getShellLines(scriptFile), "\n") + err := exitcode.WriteFileWithCheck(prepareScriptFile, []byte(prepareScriptContent), os.ModePerm) + if err != nil { + return "", err + } else { + return prepareScriptFile, nil + } + } +} + +// getShellLines 根据不同的shell的参数要求，这里可能需要不同的参数或者参数顺序 +func getShellLines(scriptFile string) (newLines []string) { + shell := getCurrentShell() + switch shell { + case "/bin/tcsh": + newLines = []string{ + "#!" + shell, + "exec " + shell + " " + scriptFile + " -l", + } + default: + newLines = []string{ + "#!" + shell, + "exec " + shell + " -l " + scriptFile, + } + } + return newLines +} + +func getCurrentShell() (shell string) { + if config.GAgentConfig.DetectShell { + shell = os.Getenv("SHELL") + if strings.TrimSpace(shell) == "" { + shell = "/bin/bash" + } + } else { + shell = "/bin/bash" + } + logs.Info("current shell: ", shell) + return +} + +func StartProcessCmd( + command string, + args []string, + workDir string, + envMap map[string]string, + runUser string, + enableExitGroup bool, +) (*exec.Cmd, error) { + cmd := exec.Command(command) + + // arm64机器目前无法通过worker杀进程 + if enableExitGroup { + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + } + + if len(args) > 0 { + cmd.Args = append(cmd.Args, args...) + } + + if workDir != "" { + cmd.Dir = workDir + } + + cmd.Env = os.Environ() + if envMap != nil { + for k, v := range envMap { + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v)) + } + } + + err := ucommand.SetUser(cmd, runUser) + if err != nil { + logs.Error("set user failed: ", err.Error()) + return nil, errors.Wrap(err, "Please check [devops.slave.user] in the {agent_dir}/.agent.properties") + } + + logs.Info("cmd.Path: ", cmd.Path) + logs.Info("cmd.Args: ", cmd.Args) + logs.Info("cmd.workDir: ", cmd.Dir) + logs.Info("runUser: ", runUser) + + err = cmd.Start() + if err != nil { + return nil, err + } + + return cmd, nil +} diff --git a/src/agent/agent/src/pkg/job/do_build_win.go b/src/agent/agent/src/pkg/job/do_build_win.go new file mode 100644 index 00000000000..3e49070a7ac --- /dev/null +++ b/src/agent/agent/src/pkg/job/do_build_win.go @@ -0,0 +1,193 @@ +//go:build windows +// +build windows + +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package job + +import ( + "fmt" + "github.com/pkg/errors" + "os" + "os/exec" + "syscall" + + "github.com/TencentBlueKing/bk-ci/agent/src/pkg/api" + "github.com/TencentBlueKing/bk-ci/agent/src/pkg/config" + "github.com/TencentBlueKing/bk-ci/agent/src/pkg/constant" + "github.com/TencentBlueKing/bk-ci/agent/src/pkg/i18n" + ucommand "github.com/TencentBlueKing/bk-ci/agent/src/pkg/util/command" + "github.com/TencentBlueKing/bk-ci/agent/src/pkg/util/process" + "github.com/TencentBlueKing/bk-ci/agent/src/pkg/util/systemutil" + "github.com/TencentBlueKing/bk-ci/agentcommon/logs" + "github.com/TencentBlueKing/bk-ci/agentcommon/utils/fileutil" +) + +func doBuild( + buildInfo *api.ThirdPartyBuildInfo, + tmpDir string, + workDir string, + goEnv map[string]string, + runUser string, +) error { + var err error + var exitGroup process.ProcessExitGroup + enableExitGroup := config.FetchEnvAndCheck(constant.DevopsAgentEnableExitGroup, "true") + if enableExitGroup { + logs.Info("DEVOPS_AGENT_ENABLE_EXIT_GROUP enable") + exitGroup, err = process.NewProcessExitGroup() + if err != nil { + errMsg := i18n.Localize("StartWorkerProcessFailed", map[string]interface{}{"err": err.Error()}) + logs.Error(errMsg) + workerBuildFinish(buildInfo.ToFinish(false, errMsg, api.BuildProcessStartErrorEnum)) + return err + } + + defer func() { + logs.Infof("%s exit group dispose", buildInfo.BuildId) + exitGroup.Dispose() + }() + } + + startCmd := config.GetJava() + agentLogPrefix := fmt.Sprintf("%s_%s_agent", buildInfo.BuildId, buildInfo.VmSeqId) + errorMsgFile := getWorkerErrorMsgFile(buildInfo.BuildId, buildInfo.VmSeqId) + args := []string{ + "-Djava.io.tmpdir=" + tmpDir, + "-Ddevops.agent.error.file=" + errorMsgFile, + "-Dbuild.type=AGENT", + "-DAGENT_LOG_PREFIX=" + agentLogPrefix, + "-Xmx2g", // #5806 兼容性问题，必须独立一行 + "-jar", + config.BuildAgentJarPath(), + getEncodedBuildInfo(buildInfo)} + cmd, err := StartProcessCmd(startCmd, args, workDir, goEnv, runUser) + if err != nil { + errMsg := i18n.Localize("StartWorkerProcessFailed", map[string]interface{}{"err": err.Error()}) + logs.Error(errMsg) + workerBuildFinish(buildInfo.ToFinish(false, errMsg, api.BuildProcessStartErrorEnum)) + return err + } + pid := cmd.Process.Pid + + if enableExitGroup { + logs.Infof("%s process %d add exit group ", buildInfo.BuildId, pid) + if err := exitGroup.AddProcess(cmd.Process); err != nil { + logs.Errorf("%s add process to %d exit group error %s", buildInfo.BuildId, pid, err.Error()) + } + } + + // 添加需要构建结束后删除的文件 + buildInfo.ToDelTmpFiles = []string{errorMsgFile} + + GBuildManager.AddBuild(pid, buildInfo) + logs.Info(fmt.Sprintf("[%s]|Job#_%s|Build started, pid:%d ", buildInfo.BuildId, buildInfo.VmSeqId, pid)) + + // #5806 预先录入异常信息，在构建进程正常结束时清理掉。如果没清理掉，则说明进程非正常退出，可能被OS或人为杀死 + _ = fileutil.WriteString(errorMsgFile, i18n.Localize("BuilderProcessWasKilled", nil)) + _ = systemutil.Chmod(errorMsgFile, os.ModePerm) + + err = cmd.Wait() + // #5806 从b-xxxx_build_msg.log 读取错误信息，此信息可由worker-agent.jar写入，用于当异常时能够将信息上报给服务器 + msgFile := getWorkerErrorMsgFile(buildInfo.BuildId, buildInfo.VmSeqId) + msg, _ := fileutil.GetString(msgFile) + if err != nil { + logs.Errorf("build[%s] pid[%d] finish, state=%v err=%v, msg=%s", buildInfo.BuildId, pid, cmd.ProcessState, err, msg) + } else { + logs.Infof("build[%s] pid[%d] finish, state=%v err=%v, msg=%s", buildInfo.BuildId, pid, cmd.ProcessState, err, msg) + } + + // #10362 Worker杀掉当前进程父进程导致Agent误报 + // agent 改动后可能会导致业务执行完成但是进程被杀掉导致流水线错误，所以将错误只是作为额外信息添加 + cmdErrMsg := "" + if err != nil { + cmdErrMsg = "|" + err.Error() + } + + success := true + if len(msg) == 0 { + msg = i18n.Localize("WorkerExit", map[string]interface{}{"pid": pid}) + cmdErrMsg + } else { + msg += cmdErrMsg + success = false + } + + GBuildManager.DeleteBuild(pid) + if success { + workerBuildFinish(buildInfo.ToFinish(success, msg, api.NoErrorEnum)) + } else { + workerBuildFinish(buildInfo.ToFinish(success, msg, api.BuildProcessRunErrorEnum)) + } + + return nil +} + +func StartProcessCmd(command string, args []string, workDir string, envMap map[string]string, runUser string) (*exec.Cmd, error) { + cmd := exec.Command(command) + + if config.FetchEnvAndCheck(constant.DevopsAgentEnableNewConsole, "true") { + cmd.SysProcAttr = &syscall.SysProcAttr{ + CreationFlags: constant.WinCommandNewConsole, + NoInheritHandles: true, + } + logs.Info("DEVOPS_AGENT_ENABLE_NEW_CONSOLE enabled") + } + + if len(args) > 0 { + cmd.Args = append(cmd.Args, args...) + } + + if workDir != "" { + cmd.Dir = workDir + } + + cmd.Env = os.Environ() + if envMap != nil { + for k, v := range envMap { + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v)) + } + } + + err := ucommand.SetUser(cmd, runUser) + if err != nil { + logs.Error("set user failed: ", err.Error()) + return nil, errors.Wrap(err, "Please check [devops.slave.user] in the {agent_dir}/.agent.properties") + } + + logs.Info("cmd.Path: ", cmd.Path) + logs.Info("cmd.Args: ", cmd.Args) + logs.Info("cmd.workDir: ", cmd.Dir) + logs.Info("runUser: ", runUser) + + err = cmd.Start() + if err != nil { + return nil, err + } + + return cmd, nil +} diff --git a/src/agent/agent/src/pkg/job/job.go b/src/agent/agent/src/pkg/job/job.go deleted file mode 100644 index 00ba6971cb4..00000000000 --- a/src/agent/agent/src/pkg/job/job.go +++ /dev/null @@ -1,43 +0,0 @@ -package job - -import ( - "fmt" - - "github.com/TencentBlueKing/bk-ci/agent/src/pkg/util/systemutil" -) - -const ( - errorMsgFileSuffix = "build_msg.log" - prepareStartScriptFilePrefix = "devops_agent_prepare_start" - prepareStartScriptFileSuffix = ".sh" - startScriptFilePrefix = "devops_agent_start" - startScriptFileSuffix = ".sh" -) - -// getWorkerErrorMsgFile 获取worker执行错误信息的日志文件 -func getWorkerErrorMsgFile(buildId, vmSeqId string) string { - return fmt.Sprintf("%s/build_tmp/%s_%s_%s", - systemutil.GetWorkDir(), buildId, vmSeqId, errorMsgFileSuffix) -} - -// getUnixWorkerPrepareStartScriptFile 获取unix系统，主要是darwin和linux的prepare start script文件 -func getUnixWorkerPrepareStartScriptFile(projectId, buildId, vmSeqId string) string { - return fmt.Sprintf("%s/%s_%s_%s_%s%s", - systemutil.GetWorkDir(), prepareStartScriptFilePrefix, projectId, buildId, vmSeqId, prepareStartScriptFileSuffix) -} - -// getUnixWorkerStartScriptFile 获取unix系统，主要是darwin和linux的prepare start script文件 -func getUnixWorkerStartScriptFile(projectId, buildId, vmSeqId string) string { - return fmt.Sprintf("%s/%s_%s_%s_%s%s", - systemutil.GetWorkDir(), startScriptFilePrefix, projectId, buildId, vmSeqId, startScriptFileSuffix) -} - -// 校验当前是否有正在跑的任务 -func CheckRunningJob() bool { - if GBuildManager.GetPreInstancesCount() > 0 || - GBuildManager.GetInstanceCount() > 0 || - GBuildDockerManager.GetInstanceCount() > 0 { - return true - } - return false -} diff --git a/src/agent/agent/src/pkg/job_docker/job_docker.go b/src/agent/agent/src/pkg/job_docker/job_docker.go index 51de755e01a..6448fdff416 100644 --- a/src/agent/agent/src/pkg/job_docker/job_docker.go +++ b/src/agent/agent/src/pkg/job_docker/job_docker.go @@ -5,11 +5,11 @@ import ( "encoding/base64" "encoding/json" "fmt" + "github.com/docker/docker/api/types/registry" "os" "strings" "github.com/TencentBlueKing/bk-ci/agent/src/pkg/api" - "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" @@ -20,7 +20,7 @@ import ( const ( LocalDockerBuildTmpDirName = "docker_build_tmp" LocalDockerWorkSpaceDirName = "docker_workspace" - DockerLogDir = "/data/logs" + DockerLogDir = "/data/devops/logs" ) type DockerHostInfo struct { @@ -29,7 +29,7 @@ type DockerHostInfo struct { type ImagePullInfo struct { ImageName string - AuthType types.AuthConfig + AuthType registry.AuthConfig } type ContainerCreateInfo struct { @@ -40,7 +40,7 @@ type ContainerCreateInfo struct { } func parseApiDockerOptions(o api.DockerOptions) []string { - args := []string{} + var args []string if o.Volumes != nil && len(o.Volumes) > 0 { for _, v := range o.Volumes { args = append(args, "--volume", strings.TrimSpace(v)) @@ -57,10 +57,14 @@ func parseApiDockerOptions(o api.DockerOptions) []string { args = append(args, "--gpus", strings.TrimSpace(o.Gpus)) } + if o.Privileged != false { + args = append(args, "--privileged") + } + return args } -func ParseDockeroptions(dockerClient *client.Client, userOptions api.DockerOptions) (*ContainerConfig, error) { +func ParseDockerOptions(dockerClient *client.Client, userOptions api.DockerOptions) (*ContainerConfig, error) { // 将指定好的options直接换成args argv := parseApiDockerOptions(userOptions) if len(argv) == 0 { @@ -68,9 +72,9 @@ func ParseDockeroptions(dockerClient *client.Client, userOptions api.DockerOptio } // 解析args为flagSet - flagset := pflag.NewFlagSet(os.Args[0], pflag.ContinueOnError) - copts := addFlags(flagset) - err := flagset.Parse(argv) + flagSet := pflag.NewFlagSet(os.Args[0], pflag.ContinueOnError) + copts := addFlags(flagSet) + err := flagSet.Parse(argv) if err != nil { errMsg := fmt.Sprintf("解析用户docker options失败: %s", err.Error()) return nil, errors.New(errMsg) @@ -84,7 +88,7 @@ func ParseDockeroptions(dockerClient *client.Client, userOptions api.DockerOptio } // 解析配置为可用docker配置, 目前只有linux支持，所以只使用linux相关配置 - containerConfig, err := parse(flagset, copts, ping.OSType) + containerConfig, err := parse(flagSet, copts, ping.OSType) if err != nil { errMsg := fmt.Sprintf("解析用户docker options 为docker配置 错误: %s", err.Error()) return nil, errors.New(errMsg) @@ -93,9 +97,9 @@ func ParseDockeroptions(dockerClient *client.Client, userOptions api.DockerOptio return containerConfig, nil } -// policy 为空，并且容器镜像的标签是 :latest， image-pull-policy 会自动设置为 always +// IfPullImage policy 为空，并且容器镜像的标签是 :latest， image-pull-policy 会自动设置为 always // policy 为空，并且为容器镜像指定了非 :latest 的标签， image-pull-policy 就会自动设置为 if-not-present -func IfPullImage(localExist, islatest bool, policy string) bool { +func IfPullImage(localExist, isLatest bool, policy string) bool { // 为空和枚举写错走一套逻辑 switch policy { case api.ImagePullPolicyAlways.String(): @@ -107,7 +111,7 @@ func IfPullImage(localExist, islatest bool, policy string) bool { return false } default: - if islatest { + if isLatest { return true } else { if !localExist { @@ -125,7 +129,7 @@ func GenerateDockerAuth(user, pass string) (string, error) { return "", nil } - authConfig := types.AuthConfig{ + authConfig := registry.AuthConfig{ Username: user, Password: pass, } diff --git a/src/agent/agent/src/pkg/job_docker/opts.go b/src/agent/agent/src/pkg/job_docker/opts.go index f4f390451c6..a14e38f935b 100644 --- a/src/agent/agent/src/pkg/job_docker/opts.go +++ b/src/agent/agent/src/pkg/job_docker/opts.go @@ -324,7 +324,7 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*Con // Validate the input mac address if copts.macAddress != "" { if _, err := opts.ValidateMACAddress(copts.macAddress); err != nil { - return nil, errors.Errorf("%s is not a valid mac address", copts.macAddress) + return nil, fmt.Errorf("%s is not a valid mac address", copts.macAddress) } } if copts.stdin { @@ -340,7 +340,7 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*Con swappiness := copts.swappiness if swappiness != -1 && (swappiness < 0 || swappiness > 100) { - return nil, errors.Errorf("invalid value: %d. Valid memory swappiness range is 0-100", swappiness) + return nil, fmt.Errorf("invalid value: %d. Valid memory swappiness range is 0-100", swappiness) } mounts := copts.mounts.Value() @@ -419,7 +419,7 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*Con // Merge in exposed ports to the map of published ports for _, e := range copts.expose.GetAll() { if strings.Contains(e, ":") { - return nil, errors.Errorf("invalid port format for --expose: %s", e) + return nil, fmt.Errorf("invalid port format for --expose: %s", e) } // support two formats for expose, original format /[] // or /[] @@ -428,7 +428,7 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*Con // if expose a port, the start and end port are the same start, end, err := nat.ParsePortRange(port) if err != nil { - return nil, errors.Errorf("invalid range format for --expose: %s, error: %s", e, err) + return nil, fmt.Errorf("invalid range format for --expose: %s, error: %s", e, err) } for i := start; i <= end; i++ { p, err := nat.NewPort(proto, strconv.FormatUint(i, 10)) @@ -477,22 +477,22 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*Con pidMode := container.PidMode(copts.pidMode) if !pidMode.Valid() { - return nil, errors.Errorf("--pid: invalid PID mode") + return nil, errors.New("--pid: invalid PID mode") } utsMode := container.UTSMode(copts.utsMode) if !utsMode.Valid() { - return nil, errors.Errorf("--uts: invalid UTS mode") + return nil, errors.New("--uts: invalid UTS mode") } usernsMode := container.UsernsMode(copts.usernsMode) if !usernsMode.Valid() { - return nil, errors.Errorf("--userns: invalid USER mode") + return nil, errors.New("--userns: invalid USER mode") } cgroupnsMode := container.CgroupnsMode(copts.cgroupnsMode) if !cgroupnsMode.Valid() { - return nil, errors.Errorf("--cgroupns: invalid CGROUP mode") + return nil, errors.New("--cgroupns: invalid CGROUP mode") } restartPolicy, err := opts.ParseRestartPolicy(copts.restartPolicy) @@ -526,7 +526,7 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*Con copts.healthRetries != 0 if copts.noHealthcheck { if haveHealthSettings { - return nil, errors.Errorf("--no-healthcheck conflicts with --health-* options") + return nil, errors.New("--no-healthcheck conflicts with --health-* options") } healthConfig = &container.HealthConfig{Test: strslice.StrSlice{"NONE"}} } else if haveHealthSettings { @@ -535,16 +535,16 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*Con probe = []string{"CMD-SHELL", copts.healthCmd} } if copts.healthInterval < 0 { - return nil, errors.Errorf("--health-interval cannot be negative") + return nil, errors.New("--health-interval cannot be negative") } if copts.healthTimeout < 0 { - return nil, errors.Errorf("--health-timeout cannot be negative") + return nil, errors.New("--health-timeout cannot be negative") } if copts.healthRetries < 0 { - return nil, errors.Errorf("--health-retries cannot be negative") + return nil, errors.New("--health-retries cannot be negative") } if copts.healthStartPeriod < 0 { - return nil, fmt.Errorf("--health-start-period cannot be negative") + return nil, errors.New("--health-start-period cannot be negative") } healthConfig = &container.HealthConfig{ @@ -660,7 +660,7 @@ func parse(flags *pflag.FlagSet, copts *containerOptions, serverOS string) (*Con } if copts.autoRemove && !hostConfig.RestartPolicy.IsNone() { - return nil, errors.Errorf("Conflicting options: --restart and --rm") + return nil, errors.New("Conflicting options: --restart and --rm") } // only set this value if the user provided the flag, else it should default to nil @@ -717,7 +717,7 @@ func parseNetworkOpts(copts *containerOptions) (map[string]*networktypes.Endpoin return nil, err } if _, ok := endpoints[n.Target]; ok { - return nil, errdefs.InvalidParameter(errors.Errorf("network %q is specified multiple times", n.Target)) + return nil, errdefs.InvalidParameter(fmt.Errorf("network %q is specified multiple times", n.Target)) } // For backward compatibility: if no custom options are provided for the network, @@ -814,7 +814,7 @@ func convertToStandardNotation(ports []string) ([]string, error) { for _, param := range strings.Split(publish, ",") { k, v, ok := strings.Cut(param, "=") if !ok || k == "" { - return optsList, errors.Errorf("invalid publish opts format (should be name=value but got '%s')", param) + return optsList, fmt.Errorf("invalid publish opts format (should be name=value but got '%s')", param) } params[k] = v } @@ -829,7 +829,7 @@ func convertToStandardNotation(ports []string) ([]string, error) { func parseLoggingOpts(loggingDriver string, loggingOpts []string) (map[string]string, error) { loggingOptsMap := opts.ConvertKVStringsToMap(loggingOpts) if loggingDriver == "none" && len(loggingOpts) > 0 { - return map[string]string{}, errors.Errorf("invalid logging opts for driver %s", loggingDriver) + return map[string]string{}, fmt.Errorf("invalid logging opts for driver %s", loggingDriver) } return loggingOptsMap, nil } @@ -843,16 +843,16 @@ func parseSecurityOpts(securityOpts []string) ([]string, error) { } if (!ok || v == "") && k != "no-new-privileges" { // "no-new-privileges" is the only option that does not require a value. - return securityOpts, errors.Errorf("Invalid --security-opt: %q", opt) + return securityOpts, fmt.Errorf("Invalid --security-opt: %q", opt) } if k == "seccomp" && v != "unconfined" { f, err := os.ReadFile(v) if err != nil { - return securityOpts, errors.Errorf("opening seccomp profile (%s) failed: %v", v, err) + return securityOpts, errors.Wrapf(err, "opening seccomp profile (%s) failed", v) } b := bytes.NewBuffer(nil) if err := json.Compact(b, f); err != nil { - return securityOpts, errors.Errorf("compacting json for seccomp profile (%s) failed: %v", v, err) + return securityOpts, errors.Wrapf(err, "compacting json for seccomp profile (%s) failed", v) } securityOpts[key] = fmt.Sprintf("seccomp=%s", b.Bytes()) } @@ -886,7 +886,7 @@ func parseStorageOpts(storageOpts []string) (map[string]string, error) { for _, option := range storageOpts { k, v, ok := strings.Cut(option, "=") if !ok { - return nil, errors.Errorf("invalid storage option") + return nil, errors.New("invalid storage option") } m[k] = v } @@ -901,7 +901,7 @@ func parseDevice(device, serverOS string) (container.DeviceMapping, error) { case "windows": return parseWindowsDevice(device) } - return container.DeviceMapping{}, errors.Errorf("unknown server OS: %s", serverOS) + return container.DeviceMapping{}, fmt.Errorf("unknown server OS: %s", serverOS) } // parseLinuxDevice parses a device mapping string to a container.DeviceMapping struct @@ -925,7 +925,7 @@ func parseLinuxDevice(device string) (container.DeviceMapping, error) { case 1: src = arr[0] default: - return container.DeviceMapping{}, errors.Errorf("invalid device specification: %s", device) + return container.DeviceMapping{}, fmt.Errorf("invalid device specification: %s", device) } if dst == "" { @@ -955,7 +955,7 @@ func validateDeviceCgroupRule(val string) (string, error) { return val, nil } - return val, errors.Errorf("invalid device cgroup format '%s'", val) + return val, fmt.Errorf("invalid device cgroup format '%s'", val) } // validDeviceMode checks if the mode for device is valid or not. @@ -987,7 +987,7 @@ func validateDevice(val string, serverOS string) (string, error) { // Windows does validation entirely server-side return val, nil } - return "", errors.Errorf("unknown server OS: %s", serverOS) + return "", fmt.Errorf("unknown server OS: %s", serverOS) } // validateLinuxPath is the implementation of validateDevice knowing that the @@ -1002,12 +1002,12 @@ func validateLinuxPath(val string, validator func(string) bool) (string, error) var mode string if strings.Count(val, ":") > 2 { - return val, errors.Errorf("bad format for path: %s", val) + return val, fmt.Errorf("bad format for path: %s", val) } split := strings.SplitN(val, ":", 3) if split[0] == "" { - return val, errors.Errorf("bad format for path: %s", val) + return val, fmt.Errorf("bad format for path: %s", val) } switch len(split) { case 1: @@ -1026,13 +1026,13 @@ func validateLinuxPath(val string, validator func(string) bool) (string, error) containerPath = split[1] mode = split[2] if isValid := validator(split[2]); !isValid { - return val, errors.Errorf("bad mode specified: %s", mode) + return val, fmt.Errorf("bad mode specified: %s", mode) } val = fmt.Sprintf("%s:%s:%s", split[0], containerPath, mode) } if !path.IsAbs(containerPath) { - return val, errors.Errorf("%s is not an absolute path", containerPath) + return val, fmt.Errorf("%s is not an absolute path", containerPath) } return val, nil } @@ -1045,5 +1045,5 @@ func validateAttach(val string) (string, error) { return s, nil } } - return val, errors.Errorf("valid streams are STDIN, STDOUT and STDERR") + return val, errors.New("valid streams are STDIN, STDOUT and STDERR") } diff --git a/src/agent/agent/src/pkg/job_docker/opts_test.go b/src/agent/agent/src/pkg/job_docker/opts_test.go index 97e243d95be..a8e14294fbc 100644 --- a/src/agent/agent/src/pkg/job_docker/opts_test.go +++ b/src/agent/agent/src/pkg/job_docker/opts_test.go @@ -2,6 +2,7 @@ package job_docker import ( "fmt" + "github.com/pkg/errors" "io" "os" "runtime" @@ -12,7 +13,6 @@ import ( "github.com/docker/docker/api/types/container" networktypes "github.com/docker/docker/api/types/network" "github.com/docker/go-connections/nat" - "github.com/pkg/errors" "github.com/spf13/pflag" "gotest.tools/v3/assert" is "gotest.tools/v3/assert/cmp" @@ -283,7 +283,7 @@ func compareRandomizedStrings(a, b, c, d string) error { if a == d && b == c { return nil } - return errors.Errorf("strings don't match") + return errors.New("strings don't match") } // Simple parse with MacAddress validation diff --git a/src/agent/agent/src/pkg/pipeline/pipeline.go b/src/agent/agent/src/pkg/pipeline/pipeline.go index edaed3a3340..94118e99849 100644 --- a/src/agent/agent/src/pkg/pipeline/pipeline.go +++ b/src/agent/agent/src/pkg/pipeline/pipeline.go @@ -155,7 +155,7 @@ func runCommandPipelineWindows(pipeline *CommandPipeline, lines []string) error output, err := command.RunCommand(scriptFile, []string{} /*args*/, systemutil.GetWorkDir(), nil) if err != nil { _, _ = api.UpdatePipelineStatus(api.NewPipelineResponse(pipeline.SeqId, StatusFailure, "run pipeline failed: "+err.Error()+"\noutput: "+string(output))) - return errors.Wrap(err, "run pipeline failed") + return errors.Wrapf(err, "run pipeline failed") } _, _ = api.UpdatePipelineStatus(api.NewPipelineResponse(pipeline.SeqId, StatusSuccess, string(output))) diff --git a/src/agent/agent/src/pkg/upgrade/operation.go b/src/agent/agent/src/pkg/upgrade/operation.go index ba079d3e98d..27e08a2ffd6 100644 --- a/src/agent/agent/src/pkg/upgrade/operation.go +++ b/src/agent/agent/src/pkg/upgrade/operation.go @@ -55,7 +55,7 @@ func UninstallAgent() { // 错误了也不退出，最少也要干掉daemon } logs.Warn("agent process exiting") - systemutil.ExitProcess(constant.DAEMON_EXIT_CODE) + systemutil.ExitProcess(constant.DaemonExitCode) } // runUninstallUpgrader 卸载的区分开，方便进行退出处理 @@ -67,7 +67,7 @@ func runUninstallUpgrader(action string) error { if !systemutil.IsWindows() { err := systemutil.Chmod(scripPath, 0777) if err != nil { - logs.Error("agentUpgrade|chmod failed: ", err.Error()) + logs.WithError(err).Error("agentUpgrade|chmod failed") return errors.New("chmod failed: ") } } @@ -76,13 +76,13 @@ func runUninstallUpgrader(action string) error { pid, err := command.StartProcess(scripPath, args, systemutil.GetWorkDir(), nil, "") if err != nil { - logs.Error("agentUpgrade|run uninstall upgrader failed: ", err.Error()) + logs.WithError(err).Error("agentUpgrade|run uninstall upgrader failed") return errors.New("run uninstall upgrader failed") } logs.Info("agentUpgrade|start uninstall process success, pid: ", pid) logs.Warn("agentUpgrade|agent uninstall process exiting") - systemutil.ExitProcess(constant.DAEMON_EXIT_CODE) + systemutil.ExitProcess(constant.DaemonExitCode) return nil } @@ -95,7 +95,7 @@ func runUpgrader(action string) error { if !systemutil.IsWindows() { err := systemutil.Chmod(scripPath, 0777) if err != nil { - logs.Error("agentUpgrade|chmod failed: ", err.Error()) + logs.WithError(err).Error("agentUpgrade|chmod failed") return errors.New("chmod failed: ") } } @@ -107,7 +107,7 @@ func runUpgrader(action string) error { pid, err := command.StartProcess(scripPath, args, systemutil.GetWorkDir(), nil, "") if err != nil { - logs.Error("agentUpgrade|run upgrader failed: ", err.Error()) + logs.WithError(err).Error("agentUpgrade|run upgrader failed") return errors.New("run upgrader failed") } logs.Info("agentUpgrade|start process success, pid: ", pid) @@ -141,7 +141,7 @@ func DoUpgradeOperation(changeItems upgradeChangeItem) error { systemutil.GetWorkDir()+"/"+config.WorkAgentFile, true) if err != nil { - logs.Error("agentUpgrade|replace work agent file failed: ", err.Error()) + logs.WithError(err).Error("agentUpgrade|replace work agent file failed") return errors.New("replace work agent file failed") } logs.Info("agentUpgrade|replace agent file done") @@ -168,12 +168,12 @@ func DoUpgradeOperation(changeItems upgradeChangeItem) error { config.GetDockerInitFilePath(), true) if err != nil { - logs.Error("agentUpgrade|replace work docker init file failed: ", err.Error()) + logs.WithError(err).Error("agentUpgrade|replace work docker init file failed") return errors.New("replace work docker init file failed") } // 授予文件可执行权限，每次升级后赋予权限可以减少直接在启动时赋予的并发赋予的情况 if err = systemutil.Chmod(config.GetDockerInitFilePath(), os.ModePerm); err != nil { - logs.Error("agentUpgrade|chmod work docker init file failed: ", err.Error()) + logs.WithError(err).Error("agentUpgrade|chmod work docker init file failed") return errors.New("chmod work docker init file failed") } @@ -210,7 +210,7 @@ func DoUpgradeJdk() error { go func() { files, err := os.ReadDir(workDir) if err != nil { - logs.Error("agentUpgrade|upgrade jdk remove old jdk file error", err) + logs.WithError(err).Error("agentUpgrade|upgrade jdk remove old jdk file error") return } for _, file := range files { @@ -218,7 +218,7 @@ func DoUpgradeJdk() error { file.Name() != jdkTmpName { err = os.RemoveAll(workDir + "/" + file.Name()) if err != nil { - logs.Error("agentUpgrade|upgrade jdk remove old jdk file error", err) + logs.WithError(err).Error("agentUpgrade|upgrade jdk remove old jdk file error") } } } diff --git a/src/agent/agent/src/pkg/upgrade/upgrade.go b/src/agent/agent/src/pkg/upgrade/upgrade.go index 847d5114bea..72b9daa5482 100644 --- a/src/agent/agent/src/pkg/upgrade/upgrade.go +++ b/src/agent/agent/src/pkg/upgrade/upgrade.go @@ -28,6 +28,7 @@ package upgrade import ( + "github.com/pkg/errors" "os" "strings" "sync/atomic" @@ -38,8 +39,6 @@ import ( "github.com/TencentBlueKing/bk-ci/agent/src/pkg/upgrade/download" "github.com/TencentBlueKing/bk-ci/agent/src/pkg/util/command" - "github.com/pkg/errors" - "github.com/TencentBlueKing/bk-ci/agentcommon/logs" "github.com/TencentBlueKing/bk-ci/agent/src/pkg/api" @@ -50,7 +49,7 @@ import ( var JdkVersion = &JdkVersionType{} -// JdkVersion jdk版本信息缓存 +// JdkVersionType jdk版本信息缓存 type JdkVersionType struct { JdkFileModTime time.Time // 版本信息，原子级的 []string @@ -149,7 +148,7 @@ func AgentUpgrade(upgradeItem *api.UpgradeItem, hasBuild bool) { logs.Info("agentUpgrade|download upgrade files done") err := DoUpgradeOperation(changeItems) if err != nil { - logs.Error("agentUpgrade|do upgrade operation failed", err) + logs.WithError(err).Error("agentUpgrade|do upgrade operation failed") } else { success = true } @@ -161,7 +160,7 @@ func SyncJdkVersion() error { stat, err := os.Stat(config.GAgentConfig.JdkDirPath) if err != nil { if os.IsNotExist(err) { - logs.Error("syncJdkVersion no jdk dir find", err) + logs.WithError(err).Error("syncJdkVersion no jdk dir find") // jdk版本置为空，否则会一直保持有版本的状态 JdkVersion.SetVersion([]string{}) return nil @@ -175,7 +174,7 @@ func SyncJdkVersion() error { version, err := getJdkVersion() if err != nil { // 拿取错误时直接下载新的 - logs.Error("syncJdkVersion getJdkVersion err", err) + logs.WithError(err).Error("syncJdkVersion getJdkVersion err") return nil } JdkVersion.SetVersion(version) @@ -191,7 +190,7 @@ func SyncJdkVersion() error { version, err := getJdkVersion() if err != nil { // 拿取错误时直接下载新的 - logs.Error("syncJdkVersion getJdkVersion err", err) + logs.WithError(err).Error("syncJdkVersion getJdkVersion err") JdkVersion.SetVersion([]string{}) return nil } @@ -247,7 +246,7 @@ func SyncDockerInitFileMd5() error { func getJdkVersion() ([]string, error) { jdkVersion, err := command.RunCommand(config.GetJava(), []string{"-version"}, "", nil) if err != nil { - logs.Error("agent get jdk version failed: ", err.Error()) + logs.WithError(err).Error("agent get jdk version failed") exitcode.CheckSignalJdkError(err) return nil, errors.Wrap(err, "agent get jdk version failed") } @@ -346,7 +345,7 @@ func downloadUpgradeAgent(workDir, upgradeDir string) (agentChanged bool) { logs.Info("agentUpgrade|download upgrader start") _, err := download.DownloadUpgradeFile(upgradeDir) if err != nil { - logs.Error("agentUpgrade|download upgrader failed", err) + logs.WithError(err).Error("agentUpgrade|download upgrader failed") return false } logs.Info("agentUpgrade|download upgrader done") @@ -354,7 +353,7 @@ func downloadUpgradeAgent(workDir, upgradeDir string) (agentChanged bool) { logs.Info("agentUpgrade|download daemon start") newDaemonMd5, err := download.DownloadDaemonFile(upgradeDir) if err != nil { - logs.Error("agentUpgrade|download daemon failed", err) + logs.WithError(err).Error("agentUpgrade|download daemon failed") return false } logs.Info("agentUpgrade|download daemon done") diff --git a/src/agent/agent/src/pkg/upgrader/upgrader.go b/src/agent/agent/src/pkg/upgrader/upgrader.go index 2b5b9402d52..2a0d8e20000 100644 --- a/src/agent/agent/src/pkg/upgrader/upgrader.go +++ b/src/agent/agent/src/pkg/upgrader/upgrader.go @@ -1,3 +1,5 @@ +//go:build linux || darwin + /* * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. * @@ -28,8 +30,10 @@ package upgrader import ( - "errors" "fmt" + "github.com/TencentBlueKing/bk-ci/agent/src/pkg/constant" + innerFileUtil "github.com/TencentBlueKing/bk-ci/agent/src/pkg/util/fileutil" + "github.com/pkg/errors" "os" "strconv" "time" @@ -56,7 +60,7 @@ func DoUpgradeAgent() error { totalLock := flock.New(fmt.Sprintf("%s/%s.lock", systemutil.GetRuntimeDir(), systemutil.TotalLock)) err := totalLock.Lock() if err = totalLock.Lock(); err != nil { - logs.Error("get total lock failed, exit", err.Error()) + logs.WithError(err).Error("get total lock failed, exit") return errors.New("get total lock failed") } defer func() { totalLock.Unlock() }() @@ -66,10 +70,6 @@ func DoUpgradeAgent() error { #4686 1、kill devopsDaemon进程的行为在 macos 下， 如果当前是由 launchd 启动的（比如mac重启之后，devopsDaemon会由launchd接管启动） 当upgrader进程触发kill devopsDaemon时，会导致当前upgrader进程也被系统一并停掉，所以要排除macos的进程停止操作，否则会导致升级中断 - - 2、windows 因早期daemon缺失 pid文件，在安装多个agent的机器上无法很正确的寻找到正确的进程，并且windows的启动方式较多，早期用户会使用 - 直接双击devopsDaemon.exe文件来启动，以此来保证构建进程能够正确拉起带UI的程序，所以这块无法正确查找到进程，因此暂时也不考虑windows的 - devopsDaemon.exe文件升级。 windows需要手动升级 */ if daemonChange && systemutil.IsLinux() { tryKillAgentProcess(daemonProcess) // macos 在升级后只能使用手动重启 @@ -91,18 +91,18 @@ func DoUpgradeAgent() error { if agentChange { err = replaceAgentFile(config.GetClienAgentFile()) if err != nil { - logs.Error("replace agent file failed: ", err.Error()) + logs.WithError(err).Error("replace agent file failed") } } if daemonChange { - err = replaceAgentFile(config.GetClientDaemonFile()) // #4686 如果windows下daemon进程仍然存在，则会替换失败 + err = replaceAgentFile(config.GetClientDaemonFile()) if err != nil { - logs.Error("replace daemon file failed: ", err.Error()) + logs.WithError(err).Error("replace daemon file failed") } if systemutil.IsLinux() { // #4686 如上，上面仅停止Linux的devopsDaemon进程，则也只重启动Linux的 if startErr := StartDaemon(); startErr != nil { - logs.Error("start daemon failed: ", startErr.Error()) + logs.WithError(startErr).Error("start daemon failed") return startErr } logs.Info("agent start done") @@ -142,7 +142,7 @@ func tryKillAgentProcess(processName string) { func DoUninstallAgent() error { err := UninstallAgent() if err != nil { - logs.Error("uninstall agent failed: ", err.Error()) + logs.WithError(err).Error("uninstall agent failed") return errors.New("uninstall agent failed") } return nil @@ -186,7 +186,7 @@ func StartDaemon() error { startCmd := workDir + "/" + config.GetClientDaemonFile() if err := fileutil.SetExecutable(startCmd); err != nil { - logs.Warn(fmt.Errorf("chmod daemon file failed: %v", err)) + logs.WithError(err).Warn("chmod daemon file failed") return err } @@ -199,32 +199,26 @@ func StartDaemon() error { return nil } -func StopAgent() error { - logs.Info("start stop agent") - - workDir := systemutil.GetWorkDir() - startCmd := workDir + "/" + config.GetStopScript() - output, err := command.RunCommand(startCmd, []string{} /*args*/, workDir, nil) - if err != nil { - logs.Error("run stop script failed: ", err.Error()) - logs.Error("output: ", string(output)) - return errors.New("run stop script failed") - } - logs.Info("output: ", string(output)) - return nil -} - func replaceAgentFile(fileName string) error { logs.Info("replace agent file: ", fileName) src := systemutil.GetUpgradeDir() + "/" + fileName dst := systemutil.GetWorkDir() + "/" + fileName - if _, err := fileutil.CopyFile(src, dst, true); err != nil { - logs.Warn(fmt.Sprintf("copy file %s to %s failed: %s", src, dst, err)) - return err + + // 查询 dst 的状态，如果没有的话使用预设权限\ + perm := constant.CommonFileModePerm + if stat, err := os.Stat(dst); err != nil { + logs.WithError(err).Warnf("replaceAgentFile %s stat error", dst) + } else if stat != nil { + perm = stat.Mode() } - if err := fileutil.SetExecutable(dst); err != nil { - logs.Warn(fmt.Sprintf("chmod %s file failed: %s", dst, err)) - return err + + srcFile, err := os.Open(src) + if err != nil { + return errors.Wrapf(err, "replaceAgentFile open %s error", src) + } + + if err := innerFileUtil.AtomicWriteFile(dst, srcFile, perm); err != nil { + return errors.Wrapf(err, "replaceAgentFile AtomicWriteFile %s error", dst) } return nil } diff --git a/src/agent/agent/src/pkg/upgrader/upgrader_win.go b/src/agent/agent/src/pkg/upgrader/upgrader_win.go new file mode 100644 index 00000000000..f39ac1201ba --- /dev/null +++ b/src/agent/agent/src/pkg/upgrader/upgrader_win.go @@ -0,0 +1,328 @@ +//go:build windows + +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package upgrader + +import ( + "fmt" + "github.com/TencentBlueKing/bk-ci/agent/src/pkg/constant" + innerFileUtil "github.com/TencentBlueKing/bk-ci/agent/src/pkg/util/fileutil" + "github.com/pkg/errors" + "golang.org/x/sys/windows/svc/mgr" + "os" + "os/exec" + "strconv" + "strings" + "syscall" + "time" + + "github.com/TencentBlueKing/bk-ci/agentcommon/logs" + + "github.com/TencentBlueKing/bk-ci/agent/src/pkg/config" + "github.com/TencentBlueKing/bk-ci/agent/src/pkg/util/command" + "github.com/TencentBlueKing/bk-ci/agent/src/pkg/util/systemutil" + "github.com/TencentBlueKing/bk-ci/agentcommon/utils/fileutil" + + "github.com/gofrs/flock" + + "github.com/capnspacehook/taskmaster" + "github.com/shirou/gopsutil/v4/process" +) + +type startType string + +const ( + agentProcess = "agent" + daemonProcess = "daemon" + + // 服务，执行计划，手动 + serviceStart startType = "service" + taskStart startType = "task" + manualStart startType = "manual" +) + +// DoUpgradeAgent 升级agent +// 1、通过service启动的daemon因为go本身内部注册了daemon导致权限模型有些未知问题，无法更新daemon后启动，只能更新agent +// 2、通过执行计划启动的daemon因为具有登录态，可以直接执行脚本拉起 +// 3、用户双击启动的daemon和service一样，无法更新daemon，只能更新agent +func DoUpgradeAgent() error { + logs.Info("start upgrade agent") + config.Init(false) + + totalLock := flock.New(fmt.Sprintf("%s/%s.lock", systemutil.GetRuntimeDir(), systemutil.TotalLock)) + err := totalLock.Lock() + if err = totalLock.Lock(); err != nil { + logs.WithError(err).Error("get total lock failed, exit") + return errors.New("get total lock failed") + } + defer func() { totalLock.Unlock() }() + + startT := manualStart + var winTask *taskmaster.RegisteredTask = nil + // 先查询服务 + serviceName := "devops_agent_" + config.GAgentConfig.AgentId + ok := findService(serviceName) + if ok { + startT = serviceStart + } else { + if task, taskOk := findTask(serviceName); taskOk { + winTask = task + startT = taskStart + } + } + // 理论上不可能，但是作为补充可以为后文提供逻辑依据 + if startT == taskStart && winTask == nil { + logs.Warn("win task not exist update agent") + startT = manualStart + } + + logs.Infof("agent process start by %s", startT) + + daemonChange := false + if startT == taskStart { + daemonChange, err = checkUpgradeFileChange(config.GetClientDaemonFile()) + if err != nil { + logs.WithError(err).Warn("check daemon upgrade file change failed") + } + } + daemonPid := 0 + if daemonChange { + daemonPid, err = tryKillAgentProcess(daemonProcess) + if err != nil { + logs.WithError(err).Error(fmt.Sprintf("try kill daemon process failed")) + } + } + + agentChange, err := checkUpgradeFileChange(config.GetClienAgentFile()) + if err != nil { + logs.WithError(err).Warn("check agent upgrade file change failed") + } + agentPid := 0 + if agentChange { + agentPid, err = tryKillAgentProcess(agentProcess) + if err != nil { + logs.WithError(err).Error(fmt.Sprintf("try kill agent process failed")) + } + } + + if !agentChange && !daemonChange { + logs.Info("upgrade nothing, exit") + return nil + } + + // 检查进程是否被杀掉 + daemonExist := true + agentExist := true + for i := 0; i < 15; i++ { + if daemonExist && daemonPid != 0 { + exist, err := process.PidExists(int32(daemonPid)) + if err != nil { + logs.WithError(err).Errorf("check daemon process exist failed, pid: %d", daemonPid) + } + daemonExist = exist + } + if agentExist && agentPid != 0 { + exist, err := process.PidExists(int32(agentPid)) + if err != nil { + logs.WithError(err).Errorf("check agent process exist failed, pid: %d", agentPid) + } + agentExist = exist + } + if (!daemonChange || !daemonExist) && !agentExist { + logs.Infof("wait %d seconds for agent to stop done", i+1) + break + } else if i == 14 { + logs.Errorf("upgrade daemon exist %t, agent exist %t, can't upgrade", !daemonChange || !daemonExist, agentExist) + return nil + } + logs.Infof("wait %d seconds for agent to stop", i+1) + time.Sleep(1 * time.Second) + } + + // 替换更新文件 + if agentChange { + err = replaceAgentFile(config.GetClienAgentFile()) + if err != nil { + logs.WithError(err).Error("replace agent file failed") + } + } + if daemonChange { + err = replaceAgentFile(config.GetClientDaemonFile()) + if err != nil { + logs.WithError(err).Error("replace daemon file failed") + } + } + + // 只有 daemon 被杀才启动，没被杀等待被 daemon 拉起来 + if daemonChange { + switch startT { + case taskStart: + cmd := exec.Command("cmd.exe", "/c", systemutil.GetWorkDir()+"/devopsctl.vbs") + cmd.SysProcAttr = &syscall.SysProcAttr{ + CreationFlags: constant.WinCommandNewConsole | syscall.CREATE_NEW_PROCESS_GROUP, + NoInheritHandles: true, + } + if _, err = winTask.Run(); err != nil { + return errors.Wrapf(err, "start win task failed") + } + } + } + + logs.Info("agent upgrade done, upgrade process exiting") + return nil +} + +func tryKillAgentProcess(processName string) (int, error) { + logs.Info(fmt.Sprintf("try kill %s process", processName)) + pidFile := fmt.Sprintf("%s/%s.pid", systemutil.GetRuntimeDir(), processName) + agentPid, err := fileutil.GetString(pidFile) + if err != nil { + logs.Warn(fmt.Sprintf("parse %s pid failed: %s", processName, err)) + return 0, err + } + intPid, err := strconv.Atoi(agentPid) + if err != nil { + logs.Warn(fmt.Sprintf("parse %s pid: %s failed", processName, agentPid)) + return intPid, err + } + + p, err := process.NewProcess(int32(intPid)) + if err != nil { + if errors.Is(err, process.ErrorProcessNotRunning) { + return intPid, nil + } + return intPid, errors.Wrapf(err, "get process %d failed", intPid) + } + + if err := p.Kill(); err != nil { + return intPid, errors.Wrapf(err, "kill process %d failed", intPid) + } + + return intPid, nil +} + +func DoUninstallAgent() error { + err := UninstallAgent() + if err != nil { + logs.WithError(err).Error("uninstall agent failed") + return errors.New("uninstall agent failed") + } + return nil +} + +func UninstallAgent() error { + logs.Info("start uninstall agent") + + workDir := systemutil.GetWorkDir() + startCmd := workDir + "/" + config.GetUninstallScript() + output, err := command.RunCommand(startCmd, []string{} /*args*/, workDir, nil) + if err != nil { + logs.Error("run uninstall script failed: ", err.Error()) + logs.Error("output: ", string(output)) + return errors.New("run uninstall script failed") + } + logs.Info("output: ", string(output)) + return nil +} + +func checkUpgradeFileChange(fileName string) (change bool, err error) { + oldMd5, err := fileutil.GetFileMd5(systemutil.GetWorkDir() + "/" + fileName) + if err != nil { + logs.Error(fmt.Sprintf("agentUpgrade|check %s md5 failed", fileName), err) + return false, errors.New("check old md5 failed") + } + + newMd5, err := fileutil.GetFileMd5(systemutil.GetUpgradeDir() + "/" + fileName) + if err != nil { + logs.Error(fmt.Sprintf("agentUpgrade|check %s md5 failed", fileName), err) + return false, errors.New("check new md5 failed") + } + + return oldMd5 != newMd5, nil +} + +func replaceAgentFile(fileName string) error { + logs.Info("replace agent file: ", fileName) + src := systemutil.GetUpgradeDir() + "/" + fileName + dst := systemutil.GetWorkDir() + "/" + fileName + + // 查询 dst 的状态，如果没有的话使用预设权限 + var perm os.FileMode = 0600 + if stat, err := os.Stat(dst); err != nil { + logs.WithError(err).Warnf("replaceAgentFile %s stat error", dst) + } else if stat != nil { + perm = stat.Mode() + } + logs.Infof("replaceAgentFile dst file permissions: %v", perm) + + srcFile, err := os.Open(src) + if err != nil { + return errors.Wrapf(err, "replaceAgentFile open %s error", src) + } + + if err := innerFileUtil.AtomicWriteFile(dst, srcFile, perm); err != nil { + return errors.Wrapf(err, "replaceAgentFile AtomicWriteFile %s error", dst) + } + + return nil +} + +func findService(name string) bool { + m, err := mgr.Connect() + if err != nil { + logs.WithError(err).Error("connect manager failed") + return false + } + defer m.Disconnect() + + service, err := m.OpenService(name) + if err != nil { + logs.WithError(err).Error("open manager failed") + return false + } + defer service.Close() + + return true +} + +func findTask(name string) (*taskmaster.RegisteredTask, bool) { + service, err := taskmaster.Connect() + if err != nil { + logs.WithError(err).Error("connect taskmaster failed") + return nil, false + } + + task, err := service.GetRegisteredTask("\\" + name) + if err != nil && !strings.Contains(err.Error(), "error parsing registered task") { + logs.WithError(err).Error("get registered task failed") + return nil, false + } + + return &task, true +} diff --git a/src/agent/agent/src/pkg/util/command/command.go b/src/agent/agent/src/pkg/util/command/command.go index b395eec0479..3503f382f70 100644 --- a/src/agent/agent/src/pkg/util/command/command.go +++ b/src/agent/agent/src/pkg/util/command/command.go @@ -28,8 +28,8 @@ package command import ( - "errors" "fmt" + "github.com/pkg/errors" "os" "os/exec" @@ -85,9 +85,9 @@ func StartProcess(command string, args []string, workDir string, envMap map[stri } } - err := setUser(cmd, runUser) + err := SetUser(cmd, runUser) if err != nil { - logs.Error("set user failed: ", err.Error()) + logs.WithError(err).Error("set user failed") return -1, errors.New( fmt.Sprintf("%s, Please check [devops.slave.user] in the {agent_dir}/.agent.properties", err.Error())) } diff --git a/src/agent/agent/src/pkg/util/command/user.go b/src/agent/agent/src/pkg/util/command/user.go index cb20e752865..9d54e1a6d3f 100644 --- a/src/agent/agent/src/pkg/util/command/user.go +++ b/src/agent/agent/src/pkg/util/command/user.go @@ -31,8 +31,8 @@ package command import ( - "errors" "fmt" + "github.com/pkg/errors" "os/exec" "os/user" "strconv" @@ -48,7 +48,7 @@ var envHome = "HOME" var envUser = "USER" var envLogName = "LOGNAME" -func setUser(cmd *exec.Cmd, runUser string) error { +func SetUser(cmd *exec.Cmd, runUser string) error { if len(runUser) == 0 { // 传空则直接返回 return nil @@ -77,7 +77,7 @@ func setUser(cmd *exec.Cmd, runUser string) error { rUser, err := user.Lookup(runUser) if err != nil { - logs.Error("user lookup failed, user: -", runUser, "-, error: ", err.Error()) + logs.WithError(err).Error("user lookup failed, user: -", runUser, "-, error") return errors.New("user lookup failed, user: " + runUser) } uid, _ := strconv.Atoi(rUser.Uid) diff --git a/src/agent/agent/src/pkg/util/command/user_win.go b/src/agent/agent/src/pkg/util/command/user_win.go index d37f9aca68f..b1e502fb492 100644 --- a/src/agent/agent/src/pkg/util/command/user_win.go +++ b/src/agent/agent/src/pkg/util/command/user_win.go @@ -36,7 +36,7 @@ import ( "github.com/TencentBlueKing/bk-ci/agentcommon/logs" ) -func setUser(_ *exec.Cmd, runUser string) error { +func SetUser(_ *exec.Cmd, runUser string) error { logs.Info("set user(windows): ", runUser) return nil } diff --git a/src/agent/agent/src/pkg/util/fileutil/fileutil.go b/src/agent/agent/src/pkg/util/fileutil/fileutil.go new file mode 100644 index 00000000000..ee608dd7341 --- /dev/null +++ b/src/agent/agent/src/pkg/util/fileutil/fileutil.go @@ -0,0 +1,62 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package fileutil + +import ( + "github.com/TencentBlueKing/bk-ci/agent/internal/third_party/dep/fs" + exitcode "github.com/TencentBlueKing/bk-ci/agent/src/pkg/exiterror" + "github.com/TencentBlueKing/bk-ci/agent/src/pkg/util/systemutil" + "io" + "os" + "path/filepath" +) + +func AtomicWriteFile(filename string, reader io.Reader, mode os.FileMode) error { + tempFile, err := os.CreateTemp(filepath.Split(filename)) + if err != nil { + exitcode.CheckOsIoError(filename, err) + return err + } + tempName := tempFile.Name() + + if _, err := io.Copy(tempFile, reader); err != nil { + tempFile.Close() // return value is ignored as we are already on error path + exitcode.CheckOsIoError(filename, err) + return err + } + + if err := tempFile.Close(); err != nil { + return err + } + + if err := systemutil.Chmod(tempName, mode); err != nil { + return err + } + + return fs.RenameWithFallback(tempName, filename) +} diff --git a/src/agent/agent/src/pkg/util/httputil/devops.go b/src/agent/agent/src/pkg/util/httputil/devops.go index 177cfc0a605..41b4ee23064 100644 --- a/src/agent/agent/src/pkg/util/httputil/devops.go +++ b/src/agent/agent/src/pkg/util/httputil/devops.go @@ -29,19 +29,15 @@ package httputil import ( "encoding/json" - "errors" + "github.com/TencentBlueKing/bk-ci/agent/src/pkg/constant" + "github.com/TencentBlueKing/bk-ci/agentcommon/logs" + "github.com/pkg/errors" "io" "net/http" "os" - "path/filepath" - - "github.com/TencentBlueKing/bk-ci/agent/internal/third_party/dep/fs" - - "github.com/TencentBlueKing/bk-ci/agentcommon/logs" "github.com/TencentBlueKing/bk-ci/agent/src/pkg/config" - exitcode "github.com/TencentBlueKing/bk-ci/agent/src/pkg/exiterror" - "github.com/TencentBlueKing/bk-ci/agent/src/pkg/util/systemutil" + innerFileUtil "github.com/TencentBlueKing/bk-ci/agent/src/pkg/util/fileutil" "github.com/TencentBlueKing/bk-ci/agentcommon/utils/fileutil" ) @@ -179,7 +175,7 @@ func DownloadUpgradeFile(url string, headers map[string]string, filepath string) return "", errors.New("download upgrade file failed") } - err = AtomicWriteFile(filepath, resp.Body, 0644) + err = innerFileUtil.AtomicWriteFile(filepath, resp.Body, constant.CommonFileModePerm) if err != nil { logs.Error("download upgrade file failed", err) return "", errors.New("download upgrade file failed") @@ -215,28 +211,3 @@ func writeToFile(file string, content io.Reader) error { } return nil } - -func AtomicWriteFile(filename string, reader io.Reader, mode os.FileMode) error { - tempFile, err := os.CreateTemp(filepath.Split(filename)) - if err != nil { - exitcode.CheckOsIoError(filename, err) - return err - } - tempName := tempFile.Name() - - if _, err := io.Copy(tempFile, reader); err != nil { - tempFile.Close() // return value is ignored as we are already on error path - exitcode.CheckOsIoError(filename, err) - return err - } - - if err := tempFile.Close(); err != nil { - return err - } - - if err := systemutil.Chmod(tempName, mode); err != nil { - return err - } - - return fs.RenameWithFallback(tempName, filename) -} diff --git a/src/agent/agent/src/pkg/util/httputil/httputil.go b/src/agent/agent/src/pkg/util/httputil/httputil.go index b6ba6ed56d6..deefefc7e8c 100644 --- a/src/agent/agent/src/pkg/util/httputil/httputil.go +++ b/src/agent/agent/src/pkg/util/httputil/httputil.go @@ -31,8 +31,8 @@ import ( "bytes" "context" "encoding/json" - "errors" "fmt" + "github.com/pkg/errors" "io" "net/http" "net/url" diff --git a/src/agent/agent/src/pkg/util/process/process_exit_group_win.go b/src/agent/agent/src/pkg/util/process/process_exit_group_win.go new file mode 100644 index 00000000000..2b033e9f389 --- /dev/null +++ b/src/agent/agent/src/pkg/util/process/process_exit_group_win.go @@ -0,0 +1,80 @@ +//go:build windows +// +build windows + +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package process + +import ( + "os" + "unsafe" + + "golang.org/x/sys/windows" +) + +// We use this struct to retreive process handle(which is unexported) +// from os.Process using unsafe operation. +// Source https://gist.github.com/hallazzang/76f3970bfc949831808bbebc8ca15209 +type process struct { + Pid int + Handle uintptr +} + +type ProcessExitGroup windows.Handle + +func NewProcessExitGroup() (ProcessExitGroup, error) { + handle, err := windows.CreateJobObject(nil, nil) + if err != nil { + return 0, err + } + + info := windows.JOBOBJECT_EXTENDED_LIMIT_INFORMATION{ + BasicLimitInformation: windows.JOBOBJECT_BASIC_LIMIT_INFORMATION{ + LimitFlags: windows.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, + }, + } + if _, err := windows.SetInformationJobObject( + handle, + windows.JobObjectExtendedLimitInformation, + uintptr(unsafe.Pointer(&info)), + uint32(unsafe.Sizeof(info))); err != nil { + return 0, err + } + + return ProcessExitGroup(handle), nil +} + +func (g ProcessExitGroup) Dispose() error { + return windows.CloseHandle(windows.Handle(g)) +} + +func (g ProcessExitGroup) AddProcess(p *os.Process) error { + return windows.AssignProcessToJobObject( + windows.Handle(g), + windows.Handle((*process)(unsafe.Pointer(p)).Handle)) +} diff --git a/src/agent/agent/src/pkg/util/systemutil/systemutil.go b/src/agent/agent/src/pkg/util/systemutil/systemutil.go index be8f414e2f5..325511dff36 100644 --- a/src/agent/agent/src/pkg/util/systemutil/systemutil.go +++ b/src/agent/agent/src/pkg/util/systemutil/systemutil.go @@ -28,8 +28,8 @@ package systemutil import ( - "errors" "fmt" + "github.com/pkg/errors" "net" "net/url" "os" @@ -229,7 +229,7 @@ func CheckProcess(name string) bool { processLock = flock.New(processLockFile) ok, err := processLock.TryLock() if err != nil { - logs.Errorf("failed to get process lock(%s), exit: %v", processLockFile, err) + logs.WithError(err).Errorf("failed to get process lock(%s), exit", processLockFile) return false } @@ -240,7 +240,7 @@ func CheckProcess(name string) bool { totalLock := flock.New(fmt.Sprintf("%s/%s.lock", GetRuntimeDir(), TotalLock)) if err = totalLock.Lock(); err != nil { - logs.Error("get total lock failed, exit", err.Error()) + logs.WithError(err).Error("get total lock failed, exit") return false } defer func() { @@ -248,7 +248,7 @@ func CheckProcess(name string) bool { }() if err = fileutil.WriteString(pidFile, fmt.Sprintf("%d", os.Getpid())); err != nil { - logs.Errorf("failed to save pid file(%s): %v", pidFile, err) + logs.WithError(err).Errorf("failed to save pid file(%s)", pidFile) return false } diff --git a/src/agent/common/logs/export.go b/src/agent/common/logs/export.go index 3f9d6ae7115..6ac5ee7ca61 100644 --- a/src/agent/common/logs/export.go +++ b/src/agent/common/logs/export.go @@ -51,3 +51,7 @@ func WithField(key string, value interface{}) *logrus.Entry { func WithError(err error) *logrus.Entry { return Logs.WithError(err) } + +func WithErrorNoStack(err error) *logrus.Entry { + return Logs.WithField(ErrorNoStackKey, err) +} diff --git a/src/agent/common/logs/logs.go b/src/agent/common/logs/logs.go index 5e0ec83e5ec..52b9b20566d 100644 --- a/src/agent/common/logs/logs.go +++ b/src/agent/common/logs/logs.go @@ -7,14 +7,17 @@ import ( "os" "github.com/sirupsen/logrus" - log "github.com/sirupsen/logrus" "gopkg.in/natefinch/lumberjack.v2" ) -var Logs = log.WithFields(log.Fields{}) +const ( + ErrorNoStackKey = "error_no_stack_key" +) + +var Logs *logrus.Entry func Init(filepath string, isDebug bool, logStd bool) error { - logInfo := log.WithFields(log.Fields{}) + logInfo := logrus.WithFields(logrus.Fields{}) lumLog := &lumberjack.Logger{ Filename: filepath, @@ -43,9 +46,9 @@ func Init(filepath string, isDebug bool, logStd bool) error { return nil } -// DebugInit 初始化为debug模式下的log，将日志输出到标准输出流，只是为了单元测试使用 +// UNTestDebugInit DebugInit 初始化为debug模式下的log，将日志输出到标准输出流，只是为了单元测试使用 func UNTestDebugInit() { - logInfo := log.WithFields(log.Fields{}) + logInfo := logrus.WithFields(logrus.Fields{}) logInfo.Logger.SetFormatter(&MyFormatter{}) logInfo.Logger.SetOutput(os.Stdout) logInfo.Logger.SetLevel(logrus.DebugLevel) @@ -68,6 +71,10 @@ func (m *MyFormatter) Format(entry *logrus.Entry) ([]byte, error) { b.WriteString(newLog) for k, v := range entry.Data { + if k == ErrorNoStackKey { + b.WriteString(fmt.Sprintf("|error: %v", v)) + continue + } switch v := v.(type) { case error: // Otherwise errors are ignored by `encoding/json` diff --git a/src/backend/ci/build.gradle.kts b/src/backend/ci/build.gradle.kts index 1ab5dc054c1..cc25687d70b 100644 --- a/src/backend/ci/build.gradle.kts +++ b/src/backend/ci/build.gradle.kts @@ -1,3 +1,5 @@ +import java.net.URI + plugins { id("com.tencent.devops.boot") version "0.0.7" detektCheck @@ -24,6 +26,11 @@ allprojects { } } + // 新增maven 仓库 + repositories { + add(maven { url = URI("https://repo.jenkins-ci.org/releases") }) + } + // 版本管理 dependencyManagement { setApplyMavenExclusions(false) diff --git a/src/backend/ci/core/artifactory/api-artifactory/src/main/kotlin/com/tencent/devops/artifactory/api/service/OpenArtifactoryResource.kt b/src/backend/ci/core/artifactory/api-artifactory/src/main/kotlin/com/tencent/devops/artifactory/api/service/OpenArtifactoryResource.kt index 6da9812f062..f7a18e235c5 100644 --- a/src/backend/ci/core/artifactory/api-artifactory/src/main/kotlin/com/tencent/devops/artifactory/api/service/OpenArtifactoryResource.kt +++ b/src/backend/ci/core/artifactory/api-artifactory/src/main/kotlin/com/tencent/devops/artifactory/api/service/OpenArtifactoryResource.kt @@ -29,9 +29,9 @@ package com.tencent.devops.artifactory.api.service import com.tencent.bkrepo.webhook.pojo.payload.node.NodeCreatedEventPayload import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_BK_TOKEN -import io.swagger.v3.oas.annotations.tags.Tag import io.swagger.v3.oas.annotations.Operation import io.swagger.v3.oas.annotations.Parameter +import io.swagger.v3.oas.annotations.tags.Tag import javax.ws.rs.Consumes import javax.ws.rs.HeaderParam import javax.ws.rs.POST diff --git a/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/resources/OpenArtifactoryResourceImpl.kt b/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/resources/OpenArtifactoryResourceImpl.kt index df75ae53a0d..48de6cfaf50 100644 --- a/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/resources/OpenArtifactoryResourceImpl.kt +++ b/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/resources/OpenArtifactoryResourceImpl.kt @@ -36,6 +36,8 @@ import com.tencent.devops.common.api.util.JsonUtil import com.tencent.devops.common.client.Client import com.tencent.devops.common.client.ClientTokenService import com.tencent.devops.common.web.RestResource +import com.tencent.devops.common.web.annotation.BkApiPermission +import com.tencent.devops.common.web.constant.BkApiHandleType import com.tencent.devops.process.api.service.ServicePipelineRuntimeResource import org.slf4j.LoggerFactory import javax.ws.rs.core.Response @@ -47,6 +49,7 @@ class OpenArtifactoryResourceImpl( private val client: Client ) : OpenArtifactoryResource { + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun updateArtifactList( token: String, nodeCreatedEventPayload: NodeCreatedEventPayload diff --git a/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/service/impl/BkRepoArchiveFileServiceImpl.kt b/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/service/impl/BkRepoArchiveFileServiceImpl.kt index 5c8eb8c38cc..a44a6b2057c 100644 --- a/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/service/impl/BkRepoArchiveFileServiceImpl.kt +++ b/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/service/impl/BkRepoArchiveFileServiceImpl.kt @@ -52,6 +52,7 @@ import com.tencent.devops.artifactory.util.BkRepoUtils.toFileDetail import com.tencent.devops.artifactory.util.BkRepoUtils.toFileInfo import com.tencent.devops.artifactory.util.DefaultPathUtils import com.tencent.devops.common.api.constant.CommonMessageCode +import com.tencent.devops.common.api.constant.KEY_SHA_CONTENT import com.tencent.devops.common.api.exception.ErrorCodeException import com.tencent.devops.common.api.exception.RemoteServiceException import com.tencent.devops.common.api.pojo.Page @@ -64,6 +65,14 @@ import com.tencent.devops.common.archive.util.MimeUtil import com.tencent.devops.common.auth.api.AuthPermission import com.tencent.devops.common.auth.api.AuthResourceType import com.tencent.devops.common.service.utils.HomeHostUtil +import com.tencent.devops.process.api.service.ServicePipelineResource +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.beans.factory.annotation.Value +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty +import org.springframework.stereotype.Service +import org.springframework.web.context.request.RequestContextHolder +import org.springframework.web.context.request.ServletRequestAttributes import java.io.File import java.io.OutputStream import java.net.URLDecoder @@ -73,13 +82,6 @@ import java.time.LocalDateTime import java.time.format.DateTimeFormatter import javax.servlet.http.HttpServletResponse import javax.ws.rs.NotFoundException -import org.slf4j.LoggerFactory -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.beans.factory.annotation.Value -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty -import org.springframework.stereotype.Service -import org.springframework.web.context.request.RequestContextHolder -import org.springframework.web.context.request.ServletRequestAttributes @Service @Suppress("TooManyFunctions", "MagicNumber", "ComplexMethod") @@ -93,11 +95,12 @@ class BkRepoArchiveFileServiceImpl @Autowired constructor( private val dockerRegistry: String? = null override fun show(userId: String, projectId: String, artifactoryType: ArtifactoryType, path: String): FileDetail { - val nodeDetail = bkRepoClient.getFileDetail(userId = userId, + val nodeDetail = bkRepoClient.getFileDetail( + userId = userId, projectId = projectId, repoName = BkRepoUtils.getRepoName(artifactoryType), - path = path) - ?: throw NotFoundException("file[$projectId|$artifactoryType|$path] not found") + path = path + ) ?: throw NotFoundException("file[$projectId|$artifactoryType|$path] not found") return nodeDetail.toFileDetail() } @@ -115,7 +118,7 @@ class BkRepoArchiveFileServiceImpl @Autowired constructor( val pathSplit = file.name.split('.') val destPath = filePath ?: DefaultPathUtils.randomFileName(pathSplit[pathSplit.size - 1]) val metadata = mutableMapOf() - metadata["shaContent"] = file.inputStream().use { ShaUtils.sha1InputStream(it) } + metadata[KEY_SHA_CONTENT] = file.inputStream().use { ShaUtils.sha1InputStream(it) } props?.forEach { metadata[it.key] = it.value!! } @@ -266,11 +269,15 @@ class BkRepoArchiveFileServiceImpl @Autowired constructor( page = page ?: 1, pageSize = pageSize ?: DEFAULT_PAGE_SIZE, totalPages = 1, - records = nodeList.map { buildFileInfo(it) } + records = nodeList.map { buildFileInfo(it, getPipelineNames(nodeList), getBuildNums(nodeList)) } ) } - private fun buildFileInfo(it: QueryNodeInfo): FileInfo { + private fun buildFileInfo( + it: QueryNodeInfo, + pipelineNameMap: Map, + buildNumMap: Map + ): FileInfo { return if (parseArtifactoryType(it.repoName) == ArtifactoryType.IMAGE) { val (imageName, version) = DefaultPathUtils.getImageNameAndVersion(it.fullPath) val packageVersion = bkRepoClient.getPackageVersionInfo( @@ -296,18 +303,22 @@ class BkRepoArchiveFileServiceImpl @Autowired constructor( ) } } else { - buildGenericFileInfo(it) + buildGenericFileInfo(it, pipelineNameMap, buildNumMap) } } - private fun buildGenericFileInfo(nodeInfo: QueryNodeInfo): FileInfo { + private fun buildGenericFileInfo( + nodeInfo: QueryNodeInfo, + pipelineNameMap: Map, + buildNumMap: Map + ): FileInfo { // 归档插件归档目录时，在目录多归档一个.bkci_pipeline文件, 记录归档目录的信息 return if (nodeInfo.name == ".bkci_pipeline") { FileInfo( name = nodeInfo.path.split("/").lastOrNull { it.isNotBlank() } ?: StringPool.ROOT, - fullName = nodeInfo.name, - path = nodeInfo.fullPath, - fullPath = nodeInfo.fullPath, + fullName = nodeInfo.path, + path = nodeInfo.path, + fullPath = nodeInfo.path, size = nodeInfo.size, folder = nodeInfo.folder, properties = nodeInfo.metadata?.map { m -> Property(m.key, m.value.toString()) }, @@ -318,7 +329,7 @@ class BkRepoArchiveFileServiceImpl @Autowired constructor( } else { FileInfo( name = nodeInfo.name, - fullName = nodeInfo.name, + fullName = getFullName(nodeInfo, pipelineNameMap, buildNumMap), path = nodeInfo.fullPath, fullPath = nodeInfo.fullPath, size = nodeInfo.size, @@ -331,6 +342,64 @@ class BkRepoArchiveFileServiceImpl @Autowired constructor( } } + private fun getPipelineNames(nodeList: List): Map { + val pipelineIds = mutableSetOf() + nodeList.filter { it.repoName == REPO_NAME_PIPELINE }.forEach { + val paths = it.fullPath.split("/") + if (paths.size < 3) { + logger.warn("illegal pipeline repo node fullPath: ${it.fullPath}") + return@forEach + } + pipelineIds.add(paths[1]) + } + if (pipelineIds.size == 0) { + return emptyMap() + } + return client.get(ServicePipelineResource::class) + .getPipelineNameByIds(nodeList.first().projectId, pipelineIds).data.orEmpty() + } + + private fun getBuildNums(nodeList: List): Map { + val buildIds = mutableSetOf() + nodeList.filter { it.repoName == REPO_NAME_PIPELINE }.forEach { + val paths = it.fullPath.split("/") + if (paths.size < 3) { + logger.warn("illegal pipeline repo node fullPath: ${it.fullPath}") + return@forEach + } + buildIds.add(paths[2]) + } + if (buildIds.size == 0) { + return emptyMap() + } + return client.get(ServicePipelineResource::class) + .getBuildNoByBuildIds(buildIds, nodeList.first().projectId).data.orEmpty() + } + + private fun getFullName( + nodeInfo: QueryNodeInfo, + pipelineNameMap: Map, + buildNumMap: Map + ): String { + if (nodeInfo.repoName != REPO_NAME_PIPELINE) { + return nodeInfo.fullPath + } + val paths = nodeInfo.fullPath.split("/") + if (paths.size < 3) { + logger.warn("illegal pipeline repo node fullPath: ${nodeInfo.fullPath}") + return nodeInfo.fullPath + } + val pipelineId = paths[1] + val buildId = paths[2] + val pipelineName = pipelineNameMap[pipelineId] + val buildNum = buildNumMap[buildId] + if (pipelineName.isNullOrEmpty() || buildNum.isNullOrEmpty()) { + logger.warn("illegal pipelineId or buildId: $pipelineId, $buildId") + return nodeInfo.fullPath + } + return nodeInfo.fullPath.replace("/$pipelineId/$buildId", "/$pipelineName/$buildNum") + } + override fun generateDestPath( fileType: FileTypeEnum, projectId: String, @@ -340,14 +409,18 @@ class BkRepoArchiveFileServiceImpl @Autowired constructor( ): String { val result = if (FileTypeEnum.BK_CUSTOM == fileType) { if (customFilePath.isNullOrBlank() || customFilePath.contains("..")) { - throw ErrorCodeException(errorCode = CommonMessageCode.PARAMETER_IS_NULL, - params = arrayOf("customFilePath")) + throw ErrorCodeException( + errorCode = CommonMessageCode.PARAMETER_IS_NULL, + params = arrayOf("customFilePath") + ) } customFilePath.removePrefix("/") } else { if (pipelineId.isNullOrBlank() || buildId.isNullOrBlank()) { - throw ErrorCodeException(errorCode = CommonMessageCode.PARAMETER_IS_NULL, - params = arrayOf("pipelineId or buildId")) + throw ErrorCodeException( + errorCode = CommonMessageCode.PARAMETER_IS_NULL, + params = arrayOf("pipelineId or buildId") + ) } val filePath = if (customFilePath.isNullOrBlank()) { "" @@ -382,7 +455,8 @@ class BkRepoArchiveFileServiceImpl @Autowired constructor( projectId = projectId, filePath = "/$filePath", artifactoryType = artifactoryType, - fileChannelType = fileChannelType, fullUrl = fullUrl) + fileChannelType = fileChannelType, fullUrl = fullUrl + ) } override fun getFileDownloadUrls( diff --git a/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/service/impl/DiskArchiveFileServiceImpl.kt b/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/service/impl/DiskArchiveFileServiceImpl.kt index b4071d0b6d7..dc24eaaf8b3 100644 --- a/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/service/impl/DiskArchiveFileServiceImpl.kt +++ b/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/service/impl/DiskArchiveFileServiceImpl.kt @@ -40,6 +40,7 @@ import com.tencent.devops.artifactory.pojo.enums.FileChannelTypeEnum import com.tencent.devops.artifactory.pojo.enums.FileTypeEnum import com.tencent.devops.artifactory.util.DefaultPathUtils import com.tencent.devops.common.api.constant.CommonMessageCode +import com.tencent.devops.common.api.constant.KEY_SHA_CONTENT import com.tencent.devops.common.api.exception.ErrorCodeException import com.tencent.devops.common.api.pojo.Page import com.tencent.devops.common.api.util.PageUtil @@ -383,7 +384,7 @@ class DiskArchiveFileServiceImpl : ArchiveFileServiceImpl() { uploadFileToRepo(destPath, file) val shaContent = file.inputStream().use { ShaUtils.sha1InputStream(it) } var fileProps: Map = props ?: mapOf() - fileProps = fileProps.plus("shaContent" to shaContent) + fileProps = fileProps.plus(KEY_SHA_CONTENT to shaContent) val path = destPath.substring(getBasePath().length) val fileId = UUIDUtil.generate() dslContext.transaction { t -> diff --git a/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/store/service/impl/ArchiveAtomServiceImpl.kt b/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/store/service/impl/ArchiveAtomServiceImpl.kt index 8fe2080c4b3..cb9e9db4d2e 100644 --- a/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/store/service/impl/ArchiveAtomServiceImpl.kt +++ b/src/backend/ci/core/artifactory/biz-artifactory/src/main/kotlin/com/tencent/devops/artifactory/store/service/impl/ArchiveAtomServiceImpl.kt @@ -39,6 +39,7 @@ import com.tencent.devops.artifactory.pojo.PackageFileInfo import com.tencent.devops.artifactory.pojo.ReArchiveAtomRequest import com.tencent.devops.artifactory.store.service.ArchiveAtomService import com.tencent.devops.common.api.constant.CommonMessageCode +import com.tencent.devops.common.api.constant.KEY_SHA_CONTENT import com.tencent.devops.common.api.constant.STATIC import com.tencent.devops.common.api.exception.ErrorCodeException import com.tencent.devops.common.api.pojo.Result @@ -214,7 +215,7 @@ abstract class ArchiveAtomServiceImpl : ArchiveAtomService { dslContext = context, userId = userId, fileId = fileId, - props = mapOf("shaContent" to packageFileInfo.shaContent) + props = mapOf(KEY_SHA_CONTENT to packageFileInfo.shaContent) ) } } diff --git a/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/api/service/ServiceManagerResource.kt b/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/api/service/ServiceManagerResource.kt index f6c1d24c257..c0353028065 100644 --- a/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/api/service/ServiceManagerResource.kt +++ b/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/api/service/ServiceManagerResource.kt @@ -30,8 +30,8 @@ package com.tencent.devops.auth.api.service import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_BK_TOKEN import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_USER_ID import com.tencent.devops.common.api.pojo.Result -import io.swagger.v3.oas.annotations.tags.Tag import io.swagger.v3.oas.annotations.Parameter +import io.swagger.v3.oas.annotations.tags.Tag import javax.ws.rs.Consumes import javax.ws.rs.GET import javax.ws.rs.HeaderParam diff --git a/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/api/service/ServicePermissionAuthResource.kt b/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/api/service/ServicePermissionAuthResource.kt index 76838004fd0..6b7fbc569cd 100644 --- a/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/api/service/ServicePermissionAuthResource.kt +++ b/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/api/service/ServicePermissionAuthResource.kt @@ -34,9 +34,9 @@ import com.tencent.devops.common.api.auth.AUTH_HEADER_GIT_TYPE import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.auth.api.AuthPermission import com.tencent.devops.common.auth.api.pojo.AuthResourceInstance -import io.swagger.v3.oas.annotations.tags.Tag import io.swagger.v3.oas.annotations.Operation import io.swagger.v3.oas.annotations.Parameter +import io.swagger.v3.oas.annotations.tags.Tag import javax.ws.rs.Consumes import javax.ws.rs.DELETE import javax.ws.rs.GET diff --git a/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/api/service/ServiceProjectAuthResource.kt b/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/api/service/ServiceProjectAuthResource.kt index 7bc6c362ff2..2b12f32d915 100644 --- a/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/api/service/ServiceProjectAuthResource.kt +++ b/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/api/service/ServiceProjectAuthResource.kt @@ -35,9 +35,9 @@ import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.auth.api.pojo.BKAuthProjectRolesResources import com.tencent.devops.common.auth.api.pojo.BkAuthGroup import com.tencent.devops.common.auth.api.pojo.BkAuthGroupAndUserList -import io.swagger.v3.oas.annotations.tags.Tag import io.swagger.v3.oas.annotations.Operation import io.swagger.v3.oas.annotations.Parameter +import io.swagger.v3.oas.annotations.tags.Tag import javax.ws.rs.Consumes import javax.ws.rs.GET import javax.ws.rs.HeaderParam @@ -134,6 +134,22 @@ interface ServiceProjectAuthResource { group: BkAuthGroup? = null ): Result + @GET + @Path("/{projectCode}/users/{userId}/checkUserInProjectLevelGroup") + @Operation(summary = "是否该用户在项目级别的组中") + fun checkUserInProjectLevelGroup( + @HeaderParam(AUTH_HEADER_DEVOPS_BK_TOKEN) + @Parameter(description = "认证token", required = true) + token: String, + @HeaderParam(AUTH_HEADER_GIT_TYPE) + @PathParam("userId") + @Parameter(description = "用户Id", required = true) + userId: String, + @PathParam("projectCode") + @Parameter(description = "项目Code", required = true) + projectCode: String + ): Result + @GET @Path("/{projectCode}/users/{userId}/checkProjectManager") @Operation(summary = "判断是否是项目管理员") diff --git a/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/api/service/ServiceResourceGroupResource.kt b/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/api/service/ServiceResourceGroupResource.kt index 443f652c5b6..dcbc9e6e546 100644 --- a/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/api/service/ServiceResourceGroupResource.kt +++ b/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/api/service/ServiceResourceGroupResource.kt @@ -1,6 +1,8 @@ package com.tencent.devops.auth.api.service -import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID +import com.tencent.devops.auth.pojo.dto.GroupAddDTO +import com.tencent.devops.auth.pojo.vo.GroupPermissionDetailVo +import com.tencent.devops.common.api.annotation.BkInterfaceI18n import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.auth.api.pojo.BkAuthGroup import io.swagger.v3.oas.annotations.Operation @@ -8,7 +10,7 @@ import io.swagger.v3.oas.annotations.Parameter import io.swagger.v3.oas.annotations.tags.Tag import javax.ws.rs.Consumes import javax.ws.rs.DELETE -import javax.ws.rs.HeaderParam +import javax.ws.rs.GET import javax.ws.rs.POST import javax.ws.rs.Path import javax.ws.rs.PathParam @@ -21,13 +23,23 @@ import javax.ws.rs.core.MediaType @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) interface ServiceResourceGroupResource { + @GET + @Path("/{projectCode}/{groupId}/getGroupPermissionDetail") + @Operation(summary = "查询用户组权限详情") + @BkInterfaceI18n(keyPrefixNames = ["{data[*].actionId}"], responseDataCacheFlag = true) + fun getGroupPermissionDetail( + @Parameter(description = "项目Id", required = true) + @PathParam("projectCode") + projectCode: String, + @Parameter(description = "用户组ID") + @PathParam("groupId") + groupId: Int + ): Result>> + @POST @Path("/{projectCode}/createGroupByGroupCode/") @Operation(summary = "根据groupCode添加用户组") fun createGroupByGroupCode( - @Parameter(description = "用户名", required = true) - @HeaderParam(AUTH_HEADER_USER_ID) - userId: String, @Parameter(description = "项目Id", required = true) @PathParam("projectCode") projectCode: String, @@ -39,13 +51,21 @@ interface ServiceResourceGroupResource { groupCode: BkAuthGroup ): Result + @POST + @Path("/{projectCode}/createGroup/") + @Operation(summary = "创建自定义组(不包含权限，空权限组)") + fun createGroup( + @Parameter(description = "项目Id", required = true) + @PathParam("projectCode") + projectCode: String, + @Parameter(description = "创建组DTO", required = true) + groupAddDTO: GroupAddDTO + ): Result + @DELETE @Path("/{projectCode}/deleteGroup/") @Operation(summary = "删除用户组") fun deleteGroup( - @Parameter(description = "用户名", required = true) - @HeaderParam(AUTH_HEADER_USER_ID) - userId: String, @Parameter(description = "项目Id", required = true) @PathParam("projectCode") projectCode: String, diff --git a/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/api/service/ServiceResourceMemberResource.kt b/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/api/service/ServiceResourceMemberResource.kt index e829a3fe172..67c65238e08 100644 --- a/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/api/service/ServiceResourceMemberResource.kt +++ b/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/api/service/ServiceResourceMemberResource.kt @@ -1,16 +1,15 @@ package com.tencent.devops.auth.api.service import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_BK_TOKEN -import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.auth.api.AuthResourceType import com.tencent.devops.common.auth.api.pojo.BkAuthGroup import com.tencent.devops.common.auth.api.pojo.BkAuthGroupAndUserList import com.tencent.devops.project.pojo.ProjectCreateUserInfo import com.tencent.devops.project.pojo.ProjectDeleteUserInfo -import io.swagger.v3.oas.annotations.tags.Tag import io.swagger.v3.oas.annotations.Operation import io.swagger.v3.oas.annotations.Parameter +import io.swagger.v3.oas.annotations.tags.Tag import javax.ws.rs.Consumes import javax.ws.rs.DELETE import javax.ws.rs.GET @@ -27,7 +26,6 @@ import javax.ws.rs.core.MediaType @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) interface ServiceResourceMemberResource { - /** * @param resourceType 是个枚举类型详见 AuthResourceType * @see AuthResourceType @@ -78,9 +76,6 @@ interface ServiceResourceMemberResource { @HeaderParam(AUTH_HEADER_DEVOPS_BK_TOKEN) @Parameter(description = "认证token", required = true) token: String, - @Parameter(description = "用户名", required = true) - @HeaderParam(AUTH_HEADER_USER_ID) - userId: String, @PathParam("projectCode") @Parameter(description = "项目Code", required = true) projectCode: String, @@ -95,9 +90,6 @@ interface ServiceResourceMemberResource { @HeaderParam(AUTH_HEADER_DEVOPS_BK_TOKEN) @Parameter(description = "认证token", required = true) token: String, - @Parameter(description = "用户名", required = true) - @HeaderParam(AUTH_HEADER_USER_ID) - userId: String, @PathParam("projectCode") @Parameter(description = "项目Code", required = true) projectCode: String, diff --git a/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/constant/AuthMessageCode.kt b/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/constant/AuthMessageCode.kt index 9e64cae5fed..74cee010b92 100644 --- a/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/constant/AuthMessageCode.kt +++ b/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/constant/AuthMessageCode.kt @@ -133,4 +133,6 @@ object AuthMessageCode { const val ERROR_MOA_CREDENTIAL_KEY_VERIFY_FAIL = "2121082" // MOA票据校验失败 const val ERROR_USER_NOT_BELONG_TO_THE_PROJECT = "2121083" // 用户不属于项目 + + const val ERROR_USER_INFORMATION_NOT_SYNCED = "2121090" // 请等待第二天用户信息同步后再尝试操作，因为新入职用户的信息尚未同步完成。 } diff --git a/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/pojo/dto/GroupAddDTO.kt b/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/pojo/dto/GroupAddDTO.kt new file mode 100644 index 00000000000..cf7a8adc7bf --- /dev/null +++ b/src/backend/ci/core/auth/api-auth/src/main/kotlin/com/tencent/devops/auth/pojo/dto/GroupAddDTO.kt @@ -0,0 +1,11 @@ +package com.tencent.devops.auth.pojo.dto + +import io.swagger.v3.oas.annotations.media.Schema + +@Schema(title = "添加组DTO") +data class GroupAddDTO( + @get:Schema(title = "用户组名称") + val groupName: String, + @get:Schema(title = "组描述") + val groupDesc: String +) diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/common/AuthCoreConfiguration.kt b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/common/AuthCoreConfiguration.kt index 4a1f62761a1..b9cbb69b36a 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/common/AuthCoreConfiguration.kt +++ b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/common/AuthCoreConfiguration.kt @@ -29,14 +29,12 @@ package com.tencent.devops.auth.common import com.fasterxml.jackson.databind.ObjectMapper import com.tencent.devops.auth.filter.BlackListAspect -import com.tencent.devops.auth.filter.TokenCheckFilter import com.tencent.devops.auth.refresh.dispatch.AuthRefreshDispatch import com.tencent.devops.auth.refresh.listener.AuthRefreshEventListener import com.tencent.devops.auth.service.AuthUserBlackListService import com.tencent.devops.auth.utils.HostUtils -import com.tencent.devops.common.client.ClientTokenService -import com.tencent.devops.common.event.dispatcher.pipeline.mq.MeasureEventDispatcher import com.tencent.devops.common.event.dispatcher.pipeline.mq.MQ +import com.tencent.devops.common.event.dispatcher.pipeline.mq.MeasureEventDispatcher import com.tencent.devops.common.web.mq.EXTEND_RABBIT_TEMPLATE_NAME import org.springframework.amqp.core.Binding import org.springframework.amqp.core.BindingBuilder @@ -133,9 +131,6 @@ class AuthCoreConfiguration { return container } - @Bean - fun tokenFilter(clientTokenService: ClientTokenService) = TokenCheckFilter(clientTokenService) - @Bean fun blackListAspect(authUserBlackListService: AuthUserBlackListService) = BlackListAspect(authUserBlackListService) } diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/config/RbacAuthConfiguration.kt b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/config/RbacAuthConfiguration.kt index b83ed29d2a3..a1ff7fe2a2f 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/config/RbacAuthConfiguration.kt +++ b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/config/RbacAuthConfiguration.kt @@ -82,7 +82,6 @@ import com.tencent.devops.auth.service.ResourceService import com.tencent.devops.auth.service.SuperManagerService import com.tencent.devops.auth.service.iam.MigrateCreatorFixService import com.tencent.devops.auth.service.iam.PermissionProjectService -import com.tencent.devops.auth.service.iam.PermissionResourceGroupService import com.tencent.devops.auth.service.iam.PermissionResourceMemberService import com.tencent.devops.auth.service.iam.PermissionResourceService import com.tencent.devops.auth.service.iam.PermissionService @@ -166,20 +165,26 @@ class RbacAuthConfiguration { iamV2ManagerService: V2ManagerService, authResourceService: AuthResourceService, permissionSubsetManagerService: PermissionSubsetManagerService, - permissionResourceService: PermissionResourceService, + permissionProjectService: PermissionProjectService, permissionGroupPoliciesService: PermissionGroupPoliciesService, authResourceGroupDao: AuthResourceGroupDao, dslContext: DSLContext, - permissionGradeManagerService: PermissionGradeManagerService + v2ManagerService: V2ManagerService, + rbacCacheService: RbacCacheService, + monitorSpaceService: AuthMonitorSpaceService, + authResourceGroupConfigDao: AuthResourceGroupConfigDao ) = RbacPermissionResourceGroupService( iamV2ManagerService = iamV2ManagerService, authResourceService = authResourceService, permissionSubsetManagerService = permissionSubsetManagerService, - permissionResourceService = permissionResourceService, + permissionProjectService = permissionProjectService, permissionGroupPoliciesService = permissionGroupPoliciesService, authResourceGroupDao = authResourceGroupDao, dslContext = dslContext, - permissionGradeManagerService = permissionGradeManagerService + v2ManagerService = v2ManagerService, + rbacCacheService = rbacCacheService, + monitorSpaceService = monitorSpaceService, + authResourceGroupConfigDao = authResourceGroupConfigDao ) @Bean @@ -287,7 +292,7 @@ class RbacAuthConfiguration { authResourceCodeConverter: AuthResourceCodeConverter, permissionService: PermissionService, itsmService: ItsmService, - monitorSpaceService: AuthMonitorSpaceService + deptService: DeptService ) = RbacPermissionApplyService( dslContext = dslContext, v2ManagerService = v2ManagerService, @@ -300,7 +305,7 @@ class RbacAuthConfiguration { authResourceCodeConverter = authResourceCodeConverter, permissionService = permissionService, itsmService = itsmService, - monitorSpaceService = monitorSpaceService + deptService = deptService ) @Bean @@ -398,7 +403,7 @@ class RbacAuthConfiguration { authMigrationDao: AuthMigrationDao, deptService: DeptService, permissionGroupPoliciesService: PermissionGroupPoliciesService, - groupService: PermissionResourceGroupService + permissionResourceMemberService: PermissionResourceMemberService ) = MigrateV3PolicyService( v2ManagerService = v2ManagerService, iamConfiguration = iamConfiguration, @@ -413,7 +418,7 @@ class RbacAuthConfiguration { authMigrationDao = authMigrationDao, deptService = deptService, permissionGroupPoliciesService = permissionGroupPoliciesService, - groupService = groupService + permissionResourceMemberService = permissionResourceMemberService ) @Bean @@ -428,10 +433,10 @@ class RbacAuthConfiguration { authResourceCodeConverter: AuthResourceCodeConverter, permissionService: PermissionService, rbacCacheService: RbacCacheService, - groupService: PermissionResourceGroupService, authMigrationDao: AuthMigrationDao, deptService: DeptService, - permissionGroupPoliciesService: PermissionGroupPoliciesService + permissionGroupPoliciesService: PermissionGroupPoliciesService, + permissionResourceMemberService: PermissionResourceMemberService ) = MigrateV0PolicyService( v2ManagerService = v2ManagerService, iamConfiguration = iamConfiguration, @@ -443,10 +448,10 @@ class RbacAuthConfiguration { authResourceCodeConverter = authResourceCodeConverter, permissionService = permissionService, rbacCacheService = rbacCacheService, - groupService = groupService, authMigrationDao = authMigrationDao, deptService = deptService, - permissionGroupPoliciesService = permissionGroupPoliciesService + permissionGroupPoliciesService = permissionGroupPoliciesService, + permissionResourceMemberService = permissionResourceMemberService ) @Bean @@ -491,13 +496,13 @@ class RbacAuthConfiguration { @Bean fun migratePermissionHandoverService( v2ManagerService: V2ManagerService, - groupService: PermissionResourceGroupService, + permissionResourceMemberService: PermissionResourceMemberService, authResourceGroupDao: AuthResourceGroupDao, authResourceService: AuthResourceService, dslContext: DSLContext ) = MigratePermissionHandoverService( v2ManagerService = v2ManagerService, - groupService = groupService, + permissionResourceMemberService = permissionResourceMemberService, authResourceGroupDao = authResourceGroupDao, authResourceService = authResourceService, dslContext = dslContext diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/config/RbacServiceConfiguration.kt b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/config/RbacServiceConfiguration.kt index 8740d02e7f9..d620b8c2d0a 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/config/RbacServiceConfiguration.kt +++ b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/config/RbacServiceConfiguration.kt @@ -49,6 +49,7 @@ import com.tencent.devops.auth.provider.rbac.service.RbacCacheService import com.tencent.devops.auth.service.AuthAuthorizationScopesService import com.tencent.devops.auth.service.AuthProjectUserMetricsService import com.tencent.devops.auth.service.BkHttpRequestService +import com.tencent.devops.auth.service.iam.PermissionResourceGroupService import com.tencent.devops.common.client.Client import com.tencent.devops.common.event.dispatcher.trace.TraceEventDispatcher import org.jooq.DSLContext @@ -101,7 +102,6 @@ class RbacServiceConfiguration { @Bean fun permissionGradeManagerService( client: Client, - permissionGroupPoliciesService: PermissionGroupPoliciesService, iamV2ManagerService: V2ManagerService, iamConfiguration: IamConfiguration, authMonitorSpaceDao: AuthMonitorSpaceDao, @@ -112,10 +112,10 @@ class RbacServiceConfiguration { authResourceGroupConfigDao: AuthResourceGroupConfigDao, traceEventDispatcher: TraceEventDispatcher, itsmService: ItsmService, - authAuthorizationScopesService: AuthAuthorizationScopesService + authAuthorizationScopesService: AuthAuthorizationScopesService, + permissionResourceGroupService: PermissionResourceGroupService ) = PermissionGradeManagerService( client = client, - permissionGroupPoliciesService = permissionGroupPoliciesService, iamV2ManagerService = iamV2ManagerService, iamConfiguration = iamConfiguration, authMonitorSpaceDao = authMonitorSpaceDao, @@ -126,7 +126,8 @@ class RbacServiceConfiguration { authResourceGroupConfigDao = authResourceGroupConfigDao, traceEventDispatcher = traceEventDispatcher, itsmService = itsmService, - authAuthorizationScopesService = authAuthorizationScopesService + authAuthorizationScopesService = authAuthorizationScopesService, + permissionResourceGroupService = permissionResourceGroupService ) @Bean diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/service/PermissionGradeManagerService.kt b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/service/PermissionGradeManagerService.kt index 516140fbbd0..2b1bf267987 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/service/PermissionGradeManagerService.kt +++ b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/service/PermissionGradeManagerService.kt @@ -33,10 +33,8 @@ import com.tencent.bk.sdk.iam.dto.GradeManagerApplicationCreateDTO import com.tencent.bk.sdk.iam.dto.GradeManagerApplicationUpdateDTO import com.tencent.bk.sdk.iam.dto.V2PageInfoDTO import com.tencent.bk.sdk.iam.dto.manager.AuthorizationScopes -import com.tencent.bk.sdk.iam.dto.manager.ManagerRoleGroup import com.tencent.bk.sdk.iam.dto.manager.ManagerScopes import com.tencent.bk.sdk.iam.dto.manager.dto.CreateManagerDTO -import com.tencent.bk.sdk.iam.dto.manager.dto.ManagerRoleGroupDTO import com.tencent.bk.sdk.iam.dto.manager.dto.SearchGroupDTO import com.tencent.bk.sdk.iam.dto.manager.dto.UpdateManagerDTO import com.tencent.bk.sdk.iam.service.v2.V2ManagerService @@ -51,6 +49,7 @@ import com.tencent.devops.auth.pojo.ItsmCancelApplicationInfo import com.tencent.devops.auth.provider.rbac.pojo.event.AuthResourceGroupCreateEvent import com.tencent.devops.auth.provider.rbac.pojo.event.AuthResourceGroupModifyEvent import com.tencent.devops.auth.service.AuthAuthorizationScopesService +import com.tencent.devops.auth.service.iam.PermissionResourceGroupService import com.tencent.devops.common.api.exception.ErrorCodeException import com.tencent.devops.common.api.util.PageUtil import com.tencent.devops.common.api.util.UUIDUtil @@ -77,7 +76,6 @@ import org.springframework.beans.factory.annotation.Value @Suppress("LongParameterList", "TooManyFunctions") class PermissionGradeManagerService @Autowired constructor( private val client: Client, - private val permissionGroupPoliciesService: PermissionGroupPoliciesService, private val iamV2ManagerService: V2ManagerService, private val iamConfiguration: IamConfiguration, private val authMonitorSpaceDao: AuthMonitorSpaceDao, @@ -88,7 +86,8 @@ class PermissionGradeManagerService @Autowired constructor( private val authResourceGroupConfigDao: AuthResourceGroupConfigDao, private val traceEventDispatcher: TraceEventDispatcher, private val itsmService: ItsmService, - private val authAuthorizationScopesService: AuthAuthorizationScopesService + private val authAuthorizationScopesService: AuthAuthorizationScopesService, + private val permissionResourceGroupService: PermissionResourceGroupService ) { companion object { @@ -418,73 +417,13 @@ class PermissionGradeManagerService @Autowired constructor( groupType = GroupType.DEFAULT.value ) defaultGroupConfigs.forEach { groupConfig -> - createProjectGroupByGroupCode( - projectCode = projectCode, + permissionResourceGroupService.createProjectGroupByGroupCode( + projectId = projectCode, groupCode = groupConfig.groupCode ) } } - fun createProjectGroupByGroupCode( - projectCode: String, - groupCode: String - ) { - val projectInfo = authResourceService.get( - projectCode = projectCode, - resourceType = AuthResourceType.PROJECT.value, - resourceCode = projectCode - ) - val groupConfig = authResourceGroupConfigDao.getByGroupCode( - dslContext = dslContext, - resourceType = AuthResourceType.PROJECT.value, - groupCode = groupCode - ) ?: throw ErrorCodeException( - errorCode = AuthMessageCode.DEFAULT_GROUP_CONFIG_NOT_FOUND, - defaultMessage = "group($groupCode) config not exist" - ) - val resourceGroupInfo = authResourceGroupDao.get( - dslContext = dslContext, - projectCode = projectCode, - resourceType = AuthResourceType.PROJECT.value, - resourceCode = projectCode, - groupCode = groupConfig.groupCode - ) - if (resourceGroupInfo != null) { - return - } - val name = groupConfig.groupName - val description = groupConfig.description - val managerRoleGroup = ManagerRoleGroup(name, description, false) - val managerRoleGroupDTO = ManagerRoleGroupDTO.builder() - .groups(listOf(managerRoleGroup)) - .createAttributes(false) - .syncSubjectTemplate(true) - .build() - val iamGroupId = iamV2ManagerService.batchCreateRoleGroupV2(projectInfo.relationId.toInt(), managerRoleGroupDTO) - authResourceGroupDao.create( - dslContext = dslContext, - projectCode = projectCode, - resourceType = AuthResourceType.PROJECT.value, - resourceCode = projectCode, - resourceName = projectInfo.resourceName, - iamResourceCode = projectCode, - groupCode = groupConfig.groupCode, - groupName = name, - defaultGroup = false, - relationId = iamGroupId.toString() - ) - permissionGroupPoliciesService.grantGroupPermission( - authorizationScopesStr = groupConfig.authorizationScopes, - projectCode = projectCode, - projectName = projectInfo.resourceName, - resourceType = groupConfig.resourceType, - groupCode = groupConfig.groupCode, - iamResourceCode = projectCode, - resourceName = projectInfo.resourceName, - iamGroupId = iamGroupId - ) - } - fun modifyGradeDefaultGroup( gradeManagerId: Int, projectCode: String, diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/service/RbacPermissionApplyService.kt b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/service/RbacPermissionApplyService.kt index f7ae1c35619..7897c7b490c 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/service/RbacPermissionApplyService.kt +++ b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/service/RbacPermissionApplyService.kt @@ -1,6 +1,5 @@ package com.tencent.devops.auth.provider.rbac.service -import com.tencent.bk.sdk.iam.dto.InstancesDTO import com.tencent.bk.sdk.iam.dto.V2PageInfoDTO import com.tencent.bk.sdk.iam.dto.application.ApplicationDTO import com.tencent.bk.sdk.iam.dto.manager.GroupMemberVerifyInfo @@ -11,7 +10,6 @@ import com.tencent.bk.sdk.iam.service.v2.V2ManagerService import com.tencent.devops.auth.constant.AuthI18nConstants import com.tencent.devops.auth.constant.AuthI18nConstants.ACTION_NAME_SUFFIX import com.tencent.devops.auth.constant.AuthI18nConstants.AUTH_RESOURCE_GROUP_CONFIG_GROUP_NAME_SUFFIX -import com.tencent.devops.auth.constant.AuthI18nConstants.RESOURCE_TYPE_NAME_SUFFIX import com.tencent.devops.auth.constant.AuthMessageCode import com.tencent.devops.auth.dao.AuthResourceGroupConfigDao import com.tencent.devops.auth.dao.AuthResourceGroupDao @@ -19,17 +17,15 @@ import com.tencent.devops.auth.pojo.ApplyJoinGroupFormDataInfo import com.tencent.devops.auth.pojo.ApplyJoinGroupInfo import com.tencent.devops.auth.pojo.AuthResourceInfo import com.tencent.devops.auth.pojo.ManagerRoleGroupInfo -import com.tencent.devops.auth.pojo.RelatedResourceInfo import com.tencent.devops.auth.pojo.ResourceGroupInfo import com.tencent.devops.auth.pojo.SearchGroupInfo import com.tencent.devops.auth.pojo.enum.GroupLevel import com.tencent.devops.auth.pojo.vo.ActionInfoVo import com.tencent.devops.auth.pojo.vo.AuthApplyRedirectInfoVo import com.tencent.devops.auth.pojo.vo.AuthRedirectGroupInfoVo -import com.tencent.devops.auth.pojo.vo.GroupPermissionDetailVo import com.tencent.devops.auth.pojo.vo.ManagerRoleGroupVO import com.tencent.devops.auth.pojo.vo.ResourceTypeInfoVo -import com.tencent.devops.auth.service.AuthMonitorSpaceService +import com.tencent.devops.auth.service.DeptService import com.tencent.devops.auth.service.GroupUserService import com.tencent.devops.auth.service.iam.PermissionApplyService import com.tencent.devops.auth.service.iam.PermissionService @@ -68,17 +64,11 @@ class RbacPermissionApplyService @Autowired constructor( val authResourceCodeConverter: AuthResourceCodeConverter, val permissionService: PermissionService, val itsmService: ItsmService, - val monitorSpaceService: AuthMonitorSpaceService + val deptService: DeptService ) : PermissionApplyService { @Value("\${auth.iamSystem:}") private val systemId = "" - @Value("\${monitor.register:false}") - private val registerMonitor: Boolean = false - - @Value("\${monitor.iamSystem:}") - private val monitorSystemId = "" - private val authApplyRedirectUrl = "${config.devopsHostGateway}/console/permission/apply?" + "project_code=%s&projectName=%s&resourceType=%s&resourceName=%s" + "&iamResourceCode=%s&action=%s&groupName=%s&groupId=%s&iamRelatedResourceType=%s" @@ -94,14 +84,15 @@ class RbacPermissionApplyService @Autowired constructor( return rbacCacheService.listResourceType2Action(resourceType) } - override fun listGroups( + override fun listGroupsForApply( userId: String, projectId: String, searchGroupInfo: SearchGroupInfo ): ManagerRoleGroupVO { logger.info("RbacPermissionApplyService|listGroups:searchGroupInfo=$searchGroupInfo") verifyProjectRouterTag(projectId) - + // 校验新用户信息是否同步完成 + isUserExists(userId) val projectInfo = authResourceService.get( projectCode = projectId, resourceType = AuthResourceType.PROJECT.value, @@ -159,6 +150,17 @@ class RbacPermissionApplyService @Autowired constructor( ) } + private fun isUserExists(userId: String) { + // 校验新用户信息是否同步完成 + val userExists = deptService.getUserInfo(userId = "admin", name = userId) != null + if (!userExists) { + logger.warn("user($userId) does not exist") + throw ErrorCodeException( + errorCode = AuthMessageCode.ERROR_USER_INFORMATION_NOT_SYNCED + ) + } + } + private fun buildBkIamPath( userId: String, resourceType: String?, @@ -429,73 +431,6 @@ class RbacPermissionApplyService @Autowired constructor( } } - override fun getGroupPermissionDetail(userId: String, groupId: Int): Map> { - val groupPermissionMap = mutableMapOf>() - groupPermissionMap[I18nUtil.getCodeLanMessage(AuthI18nConstants.BK_DEVOPS_NAME)] = - getGroupPermissionDetailBySystem(systemId, groupId) - if (registerMonitor) { - val monitorGroupPermissionDetail = getGroupPermissionDetailBySystem(monitorSystemId, groupId) - if (monitorGroupPermissionDetail.isNotEmpty()) { - groupPermissionMap[I18nUtil.getCodeLanMessage(AuthI18nConstants.BK_MONITOR_NAME)] = - getGroupPermissionDetailBySystem(monitorSystemId, groupId) - } - } - return groupPermissionMap - } - - private fun getGroupPermissionDetailBySystem(iamSystemId: String, groupId: Int): List { - val iamGroupPermissionDetailList = try { - v2ManagerService.getGroupPermissionDetail(groupId, iamSystemId) - } catch (e: Exception) { - throw ErrorCodeException( - errorCode = AuthMessageCode.GET_GROUP_PERMISSION_DETAIL_FAIL, - params = arrayOf(groupId.toString()), - defaultMessage = "Failed to get group($groupId) permission info" - ) - } - return iamGroupPermissionDetailList.map { detail -> - val relatedResourceTypesDTO = detail.resourceGroups[0].relatedResourceTypesDTO[0] - // 将resourceType转化为对应的资源类型名称 - buildRelatedResourceTypesName( - iamSystemId = iamSystemId, - instancesDTO = relatedResourceTypesDTO.condition[0].instances[0] - ) - val relatedResourceInfo = RelatedResourceInfo( - type = relatedResourceTypesDTO.type, - name = I18nUtil.getCodeLanMessage( - relatedResourceTypesDTO.type + RESOURCE_TYPE_NAME_SUFFIX - ), - instances = relatedResourceTypesDTO.condition[0].instances[0] - ) - val actionName = if (iamSystemId == monitorSystemId) { - monitorSpaceService.getMonitorActionName(action = detail.id) - } else { - rbacCacheService.getActionInfo(action = detail.id).actionName - } - GroupPermissionDetailVo( - actionId = detail.id, - name = actionName!!, - relatedResourceInfo = relatedResourceInfo - ) - }.sortedBy { it.relatedResourceInfo.type } - } - - private fun buildRelatedResourceTypesName(iamSystemId: String, instancesDTO: InstancesDTO) { - instancesDTO.let { - val resourceTypeName = if (iamSystemId == systemId) { - rbacCacheService.getResourceTypeInfo(it.type).name - } else { - I18nUtil.getCodeLanMessage(AuthI18nConstants.BK_MONITOR_SPACE) - } - it.name = resourceTypeName - it.path.forEach { element1 -> - element1.forEach { element2 -> - element2.typeName = resourceTypeName - } - } - } - } - override fun getRedirectInformation( userId: String, projectId: String, diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/service/RbacPermissionProjectService.kt b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/service/RbacPermissionProjectService.kt index 43f550a8e95..545079b77db 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/service/RbacPermissionProjectService.kt +++ b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/service/RbacPermissionProjectService.kt @@ -149,6 +149,15 @@ class RbacPermissionProjectService( } } + override fun checkUserInProjectLevelGroup(userId: String, projectCode: String): Boolean { + return resourceGroupMemberService.getResourceGroupMembers( + projectCode = projectCode, + resourceType = AuthResourceType.PROJECT.value, + resourceCode = projectCode, + group = null + ).contains(userId) + } + override fun checkProjectManager(userId: String, projectCode: String): Boolean { return rbacCacheService.checkProjectManager(userId, projectCode) } @@ -172,7 +181,6 @@ class RbacPermissionProjectService( logger.info("batch add project user:$userId|$projectCode|$roleCode|$members") val expiredTime = System.currentTimeMillis() / 1000 + TimeUnit.DAYS.toSeconds(expiredAt) resourceMemberService.batchAddResourceGroupMembers( - userId = userId, projectCode = projectCode, iamGroupId = iamGroupId, expiredTime = expiredTime, diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/service/RbacPermissionResourceGroupService.kt b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/service/RbacPermissionResourceGroupService.kt index 4704349b204..3fe23381ab8 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/service/RbacPermissionResourceGroupService.kt +++ b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/service/RbacPermissionResourceGroupService.kt @@ -28,31 +28,35 @@ package com.tencent.devops.auth.provider.rbac.service -import com.tencent.bk.sdk.iam.constants.ManagerScopesEnum +import com.tencent.bk.sdk.iam.dto.InstancesDTO import com.tencent.bk.sdk.iam.dto.V2PageInfoDTO -import com.tencent.bk.sdk.iam.dto.manager.ManagerMember import com.tencent.bk.sdk.iam.dto.manager.ManagerRoleGroup -import com.tencent.bk.sdk.iam.dto.manager.dto.GroupMemberRenewApplicationDTO -import com.tencent.bk.sdk.iam.dto.manager.dto.ManagerMemberGroupDTO +import com.tencent.bk.sdk.iam.dto.manager.dto.ManagerRoleGroupDTO import com.tencent.bk.sdk.iam.dto.manager.dto.SearchGroupDTO import com.tencent.bk.sdk.iam.service.v2.V2ManagerService import com.tencent.devops.auth.constant.AuthI18nConstants +import com.tencent.devops.auth.constant.AuthMessageCode import com.tencent.devops.auth.constant.AuthMessageCode.AUTH_GROUP_MEMBER_EXPIRED_DESC import com.tencent.devops.auth.constant.AuthMessageCode.ERROR_DEFAULT_GROUP_DELETE_FAIL import com.tencent.devops.auth.constant.AuthMessageCode.ERROR_DEFAULT_GROUP_RENAME_FAIL import com.tencent.devops.auth.constant.AuthMessageCode.ERROR_GROUP_NAME_TO_LONG import com.tencent.devops.auth.constant.AuthMessageCode.ERROR_GROUP_NAME_TO_SHORT import com.tencent.devops.auth.constant.AuthMessageCode.GROUP_EXIST +import com.tencent.devops.auth.dao.AuthResourceGroupConfigDao import com.tencent.devops.auth.dao.AuthResourceGroupDao -import com.tencent.devops.auth.pojo.dto.GroupMemberRenewalDTO +import com.tencent.devops.auth.pojo.RelatedResourceInfo +import com.tencent.devops.auth.pojo.dto.GroupAddDTO import com.tencent.devops.auth.pojo.dto.RenameGroupDTO import com.tencent.devops.auth.pojo.enum.GroupMemberStatus +import com.tencent.devops.auth.pojo.vo.GroupPermissionDetailVo import com.tencent.devops.auth.pojo.vo.IamGroupInfoVo import com.tencent.devops.auth.pojo.vo.IamGroupMemberInfoVo import com.tencent.devops.auth.pojo.vo.IamGroupPoliciesVo +import com.tencent.devops.auth.service.AuthMonitorSpaceService +import com.tencent.devops.auth.service.iam.PermissionProjectService import com.tencent.devops.auth.service.iam.PermissionResourceGroupService -import com.tencent.devops.auth.service.iam.PermissionResourceService import com.tencent.devops.common.api.exception.ErrorCodeException +import com.tencent.devops.common.api.exception.PermissionForbiddenException import com.tencent.devops.common.api.pojo.Pagination import com.tencent.devops.common.api.util.DateTimeUtil import com.tencent.devops.common.api.util.PageUtil @@ -61,18 +65,30 @@ import com.tencent.devops.common.web.utils.I18nUtil import org.jooq.DSLContext import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired +import org.springframework.beans.factory.annotation.Value @Suppress("LongParameterList") class RbacPermissionResourceGroupService @Autowired constructor( private val iamV2ManagerService: V2ManagerService, private val authResourceService: AuthResourceService, private val permissionSubsetManagerService: PermissionSubsetManagerService, - private val permissionResourceService: PermissionResourceService, + private val permissionProjectService: PermissionProjectService, private val permissionGroupPoliciesService: PermissionGroupPoliciesService, private val dslContext: DSLContext, private val authResourceGroupDao: AuthResourceGroupDao, - private val permissionGradeManagerService: PermissionGradeManagerService + private val v2ManagerService: V2ManagerService, + private val rbacCacheService: RbacCacheService, + private val monitorSpaceService: AuthMonitorSpaceService, + private val authResourceGroupConfigDao: AuthResourceGroupConfigDao ) : PermissionResourceGroupService { + @Value("\${auth.iamSystem:}") + private val systemId = "" + + @Value("\${monitor.register:false}") + private val registerMonitor: Boolean = false + + @Value("\${monitor.iamSystem:}") + private val monitorSystemId = "" companion object { private val logger = LoggerFactory.getLogger(RbacPermissionResourceGroupService::class.java) @@ -230,51 +246,19 @@ class RbacPermissionResourceGroupService @Autowired constructor( ) } - override fun renewal( - userId: String, - projectId: String, - resourceType: String, - groupId: Int, - memberRenewalDTO: GroupMemberRenewalDTO - ): Boolean { - logger.info("renewal group member|$userId|$projectId|$resourceType|$groupId") - val managerMemberGroupDTO = GroupMemberRenewApplicationDTO.builder() - .groupIds(listOf(groupId)) - .expiredAt(memberRenewalDTO.expiredAt) - .reason("renewal user group") - .applicant(userId).build() - iamV2ManagerService.renewalRoleGroupMemberApplication(managerMemberGroupDTO) - return true - } - - override fun deleteGroupMember( - userId: String, - projectId: String, - resourceType: String, - groupId: Int - ): Boolean { - logger.info("delete group member|$userId|$projectId|$resourceType|$groupId") - iamV2ManagerService.deleteRoleGroupMemberV2( - groupId, - ManagerScopesEnum.getType(ManagerScopesEnum.USER), - userId - ) - return true - } - override fun deleteGroup( - userId: String, + userId: String?, projectId: String, resourceType: String, groupId: Int ): Boolean { logger.info("delete group|$userId|$projectId|$resourceType|$groupId") - permissionResourceService.hasManagerPermission( - userId = userId, - projectId = projectId, - resourceType = AuthResourceType.PROJECT.value, - resourceCode = projectId - ) + if (userId != null) { + checkProjectManager( + userId = userId, + projectId = projectId + ) + } val authResourceGroup = authResourceGroupDao.getByRelationId( dslContext = dslContext, projectCode = projectId, @@ -297,29 +281,73 @@ class RbacPermissionResourceGroupService @Autowired constructor( return true } - override fun createGroupByGroupCode( + private fun checkProjectManager( userId: String, - projectId: String, - resourceType: String, - groupCode: String - ): Boolean { - logger.info("create group|$userId|$projectId|$groupCode|$resourceType") - permissionResourceService.hasManagerPermission( + projectId: String + ) { + val hasProjectManagePermission = permissionProjectService.checkProjectManager( userId = userId, - projectId = projectId, - resourceType = AuthResourceType.PROJECT.value, - resourceCode = projectId + projectCode = projectId ) - if (resourceType == AuthResourceType.PROJECT.value) { - permissionGradeManagerService.createProjectGroupByGroupCode( - projectCode = projectId, - groupCode = groupCode + if (!hasProjectManagePermission) { + throw PermissionForbiddenException( + message = I18nUtil.getCodeLanMessage(AuthMessageCode.ERROR_AUTH_NO_MANAGE_PERMISSION) ) } - return true } - override fun rename( + override fun createGroup( + projectId: String, + groupAddDTO: GroupAddDTO + ): Int { + logger.info("create group|$projectId|$groupAddDTO") + val projectInfo = authResourceService.get( + projectCode = projectId, + resourceType = AuthResourceType.PROJECT.value, + resourceCode = projectId + ) + + return createProjectGroupToIam( + projectCode = projectId, + projectName = projectInfo.resourceName, + relationId = projectInfo.relationId.toInt(), + groupCode = "custom", + groupName = groupAddDTO.groupName, + description = groupAddDTO.groupDesc + ) + } + + private fun createProjectGroupToIam( + projectCode: String, + projectName: String, + relationId: Int, + groupCode: String, + groupName: String, + description: String + ): Int { + val managerRoleGroup = ManagerRoleGroup(groupName, description, false) + val managerRoleGroupDTO = ManagerRoleGroupDTO.builder() + .groups(listOf(managerRoleGroup)) + .createAttributes(false) + .syncSubjectTemplate(true) + .build() + val iamGroupId = iamV2ManagerService.batchCreateRoleGroupV2(relationId, managerRoleGroupDTO) + authResourceGroupDao.create( + dslContext = dslContext, + projectCode = projectCode, + resourceType = AuthResourceType.PROJECT.value, + resourceCode = projectCode, + resourceName = projectName, + iamResourceCode = projectCode, + groupCode = groupCode, + groupName = groupName, + defaultGroup = false, + relationId = iamGroupId.toString() + ) + return iamGroupId + } + + override fun renameGroup( userId: String, projectId: String, resourceType: String, @@ -339,11 +367,9 @@ class RbacPermissionResourceGroupService @Autowired constructor( defaultMessage = "group name cannot be less than 5 characters" ) } - permissionResourceService.hasManagerPermission( + checkProjectManager( userId = userId, - projectId = projectId, - resourceType = AuthResourceType.PROJECT.value, - resourceCode = projectId + projectId = projectId ) val authResourceGroup = authResourceGroupDao.getByRelationId( dslContext = dslContext, @@ -363,22 +389,6 @@ class RbacPermissionResourceGroupService @Autowired constructor( return true } - override fun addGroupMember( - userId: String, - /*user 或 department*/ - memberType: String, - expiredAt: Long, - groupId: Int - ): Boolean { - val managerMember = ManagerMember(memberType, userId) - val managerMemberGroupDTO = ManagerMemberGroupDTO.builder() - .members(listOf(managerMember)) - .expiredAt(expiredAt) - .build() - iamV2ManagerService.createRoleGroupMemberV2(groupId, managerMemberGroupDTO) - return true - } - private fun checkDuplicateGroupName( projectId: String, groupName: String @@ -407,4 +417,119 @@ class RbacPermissionResourceGroupService @Autowired constructor( ) } } + + override fun getGroupPermissionDetail(groupId: Int): Map> { + val groupPermissionMap = mutableMapOf>() + groupPermissionMap[I18nUtil.getCodeLanMessage(AuthI18nConstants.BK_DEVOPS_NAME)] = + getGroupPermissionDetailBySystem(systemId, groupId) + if (registerMonitor) { + val monitorGroupPermissionDetail = getGroupPermissionDetailBySystem(monitorSystemId, groupId) + if (monitorGroupPermissionDetail.isNotEmpty()) { + groupPermissionMap[I18nUtil.getCodeLanMessage(AuthI18nConstants.BK_MONITOR_NAME)] = + getGroupPermissionDetailBySystem(monitorSystemId, groupId) + } + } + return groupPermissionMap + } + + override fun createProjectGroupByGroupCode( + projectId: String, + groupCode: String + ): Boolean { + val projectInfo = authResourceService.get( + projectCode = projectId, + resourceType = AuthResourceType.PROJECT.value, + resourceCode = projectId + ) + val groupConfig = authResourceGroupConfigDao.getByGroupCode( + dslContext = dslContext, + resourceType = AuthResourceType.PROJECT.value, + groupCode = groupCode + ) ?: throw ErrorCodeException( + errorCode = AuthMessageCode.DEFAULT_GROUP_CONFIG_NOT_FOUND, + defaultMessage = "group($groupCode) config not exist" + ) + val resourceGroupInfo = authResourceGroupDao.get( + dslContext = dslContext, + projectCode = projectId, + resourceType = AuthResourceType.PROJECT.value, + resourceCode = projectId, + groupCode = groupConfig.groupCode + ) + if (resourceGroupInfo != null) { + return false + } + val iamGroupId = createProjectGroupToIam( + projectCode = projectId, + projectName = projectInfo.resourceName, + relationId = projectInfo.relationId.toInt(), + groupCode = groupConfig.groupCode, + groupName = groupConfig.groupName, + description = groupConfig.description + ) + permissionGroupPoliciesService.grantGroupPermission( + authorizationScopesStr = groupConfig.authorizationScopes, + projectCode = projectId, + projectName = projectInfo.resourceName, + resourceType = groupConfig.resourceType, + groupCode = groupConfig.groupCode, + iamResourceCode = projectId, + resourceName = projectInfo.resourceName, + iamGroupId = iamGroupId + ) + return true + } + + private fun getGroupPermissionDetailBySystem(iamSystemId: String, groupId: Int): List { + val iamGroupPermissionDetailList = try { + v2ManagerService.getGroupPermissionDetail(groupId, iamSystemId) + } catch (e: Exception) { + throw ErrorCodeException( + errorCode = AuthMessageCode.GET_GROUP_PERMISSION_DETAIL_FAIL, + params = arrayOf(groupId.toString()), + defaultMessage = "Failed to get group($groupId) permission info" + ) + } + return iamGroupPermissionDetailList.map { detail -> + val relatedResourceTypesDTO = detail.resourceGroups[0].relatedResourceTypesDTO[0] + // 将resourceType转化为对应的资源类型名称 + buildRelatedResourceTypesName( + iamSystemId = iamSystemId, + instancesDTO = relatedResourceTypesDTO.condition[0].instances[0] + ) + val relatedResourceInfo = RelatedResourceInfo( + type = relatedResourceTypesDTO.type, + name = I18nUtil.getCodeLanMessage( + relatedResourceTypesDTO.type + AuthI18nConstants.RESOURCE_TYPE_NAME_SUFFIX + ), + instances = relatedResourceTypesDTO.condition[0].instances[0] + ) + val actionName = if (iamSystemId == monitorSystemId) { + monitorSpaceService.getMonitorActionName(action = detail.id) + } else { + rbacCacheService.getActionInfo(action = detail.id).actionName + } + GroupPermissionDetailVo( + actionId = detail.id, + name = actionName!!, + relatedResourceInfo = relatedResourceInfo + ) + }.sortedBy { it.relatedResourceInfo.type } + } + + private fun buildRelatedResourceTypesName(iamSystemId: String, instancesDTO: InstancesDTO) { + instancesDTO.let { + val resourceTypeName = if (iamSystemId == systemId) { + rbacCacheService.getResourceTypeInfo(it.type).name + } else { + I18nUtil.getCodeLanMessage(AuthI18nConstants.BK_MONITOR_SPACE) + } + it.name = resourceTypeName + it.path.forEach { element1 -> + element1.forEach { element2 -> + element2.typeName = resourceTypeName + } + } + } + } } diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/service/RbacPermissionResourceMemberService.kt b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/service/RbacPermissionResourceMemberService.kt index 001bc40277e..07d713b7c55 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/service/RbacPermissionResourceMemberService.kt +++ b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/service/RbacPermissionResourceMemberService.kt @@ -5,11 +5,13 @@ import com.tencent.bk.sdk.iam.dto.V2PageInfoDTO import com.tencent.bk.sdk.iam.dto.manager.ManagerMember import com.tencent.bk.sdk.iam.dto.manager.RoleGroupMemberInfo import com.tencent.bk.sdk.iam.dto.manager.V2ManagerRoleGroupInfo +import com.tencent.bk.sdk.iam.dto.manager.dto.GroupMemberRenewApplicationDTO import com.tencent.bk.sdk.iam.dto.manager.dto.ManagerMemberGroupDTO import com.tencent.bk.sdk.iam.dto.manager.dto.SearchGroupDTO import com.tencent.bk.sdk.iam.service.v2.V2ManagerService import com.tencent.devops.auth.constant.AuthMessageCode import com.tencent.devops.auth.dao.AuthResourceGroupDao +import com.tencent.devops.auth.pojo.dto.GroupMemberRenewalDTO import com.tencent.devops.auth.service.DeptService import com.tencent.devops.auth.service.iam.PermissionResourceMemberService import com.tencent.devops.common.api.exception.ErrorCodeException @@ -20,6 +22,7 @@ import com.tencent.devops.project.constant.ProjectMessageCode import org.apache.commons.lang3.RandomUtils import org.jooq.DSLContext import org.slf4j.LoggerFactory +import java.util.concurrent.Executors import java.util.concurrent.TimeUnit class RbacPermissionResourceMemberService constructor( @@ -87,11 +90,14 @@ class RbacPermissionResourceMemberService constructor( " managerId = $managerId | groupInfoList: $groupInfoList" ) // 3、获取组成员 - return groupInfoList.map { getUsersUnderGroup(groupInfo = it) } + return groupInfoList.map { + executorService.submit { + getUsersUnderGroup(groupInfo = it) + } + }.map { it.get() } } override fun batchAddResourceGroupMembers( - userId: String, projectCode: String, iamGroupId: Int, expiredTime: Long, @@ -131,7 +137,7 @@ class RbacPermissionResourceMemberService constructor( iamMemberInfos.add(ManagerMember(deptType, it)) } } - logger.info("batch add project user:$userId|$projectCode|$iamGroupId|$expiredTime|$iamMemberInfos") + logger.info("batch add project user:|$projectCode|$iamGroupId|$expiredTime|$iamMemberInfos") if (iamMemberInfos.isNotEmpty()) { val managerMemberGroup = ManagerMemberGroupDTO.builder().members(iamMemberInfos).expiredAt(expiredTime).build() @@ -189,13 +195,12 @@ class RbacPermissionResourceMemberService constructor( } override fun batchDeleteResourceGroupMembers( - userId: String, projectCode: String, iamGroupId: Int, members: List?, departments: List? ): Boolean { - logger.info("batch delete resource group members :$userId|$projectCode|$iamGroupId||$members|$departments") + logger.info("batch delete resource group members :|$projectCode|$iamGroupId||$members|$departments") verifyGroupBelongToProject( projectCode = projectCode, iamGroupId = iamGroupId @@ -245,8 +250,10 @@ class RbacPermissionResourceMemberService constructor( } val groupMemberInfoList = iamV2ManagerService.getRoleGroupMemberV2(groupInfo.id, pageInfoDTO).results val members = mutableListOf() + val nowTimestamp = System.currentTimeMillis() / 1000 groupMemberInfoList.forEach { memberInfo -> - if (memberInfo.type == ManagerScopesEnum.getType(ManagerScopesEnum.USER)) { + if (memberInfo.type == ManagerScopesEnum.getType(ManagerScopesEnum.USER) && + memberInfo.expiredAt > nowTimestamp) { members.add(memberInfo.id) } } @@ -348,6 +355,54 @@ class RbacPermissionResourceMemberService constructor( } } + override fun renewalGroupMember( + userId: String, + projectCode: String, + resourceType: String, + groupId: Int, + memberRenewalDTO: GroupMemberRenewalDTO + ): Boolean { + logger.info("renewal group member|$userId|$projectCode|$resourceType|$groupId") + val managerMemberGroupDTO = GroupMemberRenewApplicationDTO.builder() + .groupIds(listOf(groupId)) + .expiredAt(memberRenewalDTO.expiredAt) + .reason("renewal user group") + .applicant(userId).build() + iamV2ManagerService.renewalRoleGroupMemberApplication(managerMemberGroupDTO) + return true + } + + override fun deleteGroupMember( + userId: String, + projectCode: String, + resourceType: String, + groupId: Int + ): Boolean { + logger.info("delete group member|$userId|$projectCode|$resourceType|$groupId") + iamV2ManagerService.deleteRoleGroupMemberV2( + groupId, + ManagerScopesEnum.getType(ManagerScopesEnum.USER), + userId + ) + return true + } + + override fun addGroupMember( + userId: String, + /*user 或 department*/ + memberType: String, + expiredAt: Long, + groupId: Int + ): Boolean { + val managerMember = ManagerMember(memberType, userId) + val managerMemberGroupDTO = ManagerMemberGroupDTO.builder() + .members(listOf(managerMember)) + .expiredAt(expiredAt) + .build() + iamV2ManagerService.createRoleGroupMemberV2(groupId, managerMemberGroupDTO) + return true + } + companion object { private val logger = LoggerFactory.getLogger(RbacPermissionResourceMemberService::class.java) @@ -359,5 +414,7 @@ class RbacPermissionResourceMemberService constructor( // 自动续期默认180天 private val AUTO_RENEWAL_EXPIRED_AT = TimeUnit.DAYS.toSeconds(180) + + private val executorService = Executors.newFixedThreadPool(30) } } diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/service/migrate/MigratePermissionHandoverService.kt b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/service/migrate/MigratePermissionHandoverService.kt index 7344a746868..7bffe471d60 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/service/migrate/MigratePermissionHandoverService.kt +++ b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/service/migrate/MigratePermissionHandoverService.kt @@ -32,7 +32,7 @@ import com.tencent.bk.sdk.iam.service.v2.V2ManagerService import com.tencent.devops.auth.dao.AuthResourceGroupDao import com.tencent.devops.auth.pojo.dto.PermissionHandoverDTO import com.tencent.devops.auth.provider.rbac.service.AuthResourceService -import com.tencent.devops.auth.service.iam.PermissionResourceGroupService +import com.tencent.devops.auth.service.iam.PermissionResourceMemberService import com.tencent.devops.common.auth.api.AuthResourceType import com.tencent.devops.common.auth.api.pojo.DefaultGroupType import org.jboss.logging.Logger @@ -40,7 +40,7 @@ import org.jooq.DSLContext class MigratePermissionHandoverService constructor( private val v2ManagerService: V2ManagerService, - private val groupService: PermissionResourceGroupService, + private val permissionResourceMemberService: PermissionResourceMemberService, private val authResourceGroupDao: AuthResourceGroupDao, private val authResourceService: AuthResourceService, private val dslContext: DSLContext @@ -59,7 +59,7 @@ class MigratePermissionHandoverService constructor( groupCode = DefaultGroupType.MANAGER.value ) handoverToList.forEach { handoverTo -> - groupService.addGroupMember( + permissionResourceMemberService.addGroupMember( userId = handoverTo, memberType = USER_TYPE, expiredAt = GROUP_EXPIRED_TIME, @@ -90,7 +90,7 @@ class MigratePermissionHandoverService constructor( groupCode = DefaultGroupType.MANAGER.value ) try { - groupService.addGroupMember( + permissionResourceMemberService.addGroupMember( userId = handoverTo, memberType = USER_TYPE, expiredAt = GROUP_EXPIRED_TIME, diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/service/migrate/MigrateV0PolicyService.kt b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/service/migrate/MigrateV0PolicyService.kt index 2c8275373e0..ef9401ee8da 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/service/migrate/MigrateV0PolicyService.kt +++ b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/service/migrate/MigrateV0PolicyService.kt @@ -46,7 +46,7 @@ import com.tencent.devops.auth.provider.rbac.service.AuthResourceCodeConverter import com.tencent.devops.auth.provider.rbac.service.PermissionGroupPoliciesService import com.tencent.devops.auth.provider.rbac.service.RbacCacheService import com.tencent.devops.auth.service.DeptService -import com.tencent.devops.auth.service.iam.PermissionResourceGroupService +import com.tencent.devops.auth.service.iam.PermissionResourceMemberService import com.tencent.devops.auth.service.iam.PermissionService import com.tencent.devops.common.api.util.PageUtil import com.tencent.devops.common.auth.api.AuthPermission @@ -69,10 +69,10 @@ class MigrateV0PolicyService constructor( private val authResourceCodeConverter: AuthResourceCodeConverter, private val permissionService: PermissionService, private val rbacCacheService: RbacCacheService, - private val groupService: PermissionResourceGroupService, private val authMigrationDao: AuthMigrationDao, private val deptService: DeptService, - private val permissionGroupPoliciesService: PermissionGroupPoliciesService + private val permissionGroupPoliciesService: PermissionGroupPoliciesService, + private val permissionResourceMemberService: PermissionResourceMemberService ) : AbMigratePolicyService( v2ManagerService = v2ManagerService, iamConfiguration = iamConfiguration, @@ -522,7 +522,7 @@ class MigrateV0PolicyService constructor( // 自定义用户组,半年或者一年过期 V0_GROUP_EXPIRED_DAY[RandomUtils.nextInt(0, 2)] } - groupService.addGroupMember( + permissionResourceMemberService.addGroupMember( userId = member.id, memberType = member.type, expiredAt = System.currentTimeMillis() / MILLISECOND + TimeUnit.DAYS.toSeconds(expiredDay) + diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/service/migrate/MigrateV3PolicyService.kt b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/service/migrate/MigrateV3PolicyService.kt index b2973cfb2cc..c2e42c529b6 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/service/migrate/MigrateV3PolicyService.kt +++ b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/rbac/service/migrate/MigrateV3PolicyService.kt @@ -44,7 +44,7 @@ import com.tencent.devops.auth.provider.rbac.service.AuthResourceCodeConverter import com.tencent.devops.auth.provider.rbac.service.PermissionGroupPoliciesService import com.tencent.devops.auth.provider.rbac.service.RbacCacheService import com.tencent.devops.auth.service.DeptService -import com.tencent.devops.auth.service.iam.PermissionResourceGroupService +import com.tencent.devops.auth.service.iam.PermissionResourceMemberService import com.tencent.devops.auth.service.iam.PermissionService import com.tencent.devops.common.auth.api.AuthResourceType import org.jooq.DSLContext @@ -76,7 +76,7 @@ class MigrateV3PolicyService constructor( private val authMigrationDao: AuthMigrationDao, private val deptService: DeptService, private val permissionGroupPoliciesService: PermissionGroupPoliciesService, - private val groupService: PermissionResourceGroupService + private val permissionResourceMemberService: PermissionResourceMemberService ) : AbMigratePolicyService( v2ManagerService = v2ManagerService, iamConfiguration = iamConfiguration, @@ -94,18 +94,25 @@ class MigrateV3PolicyService constructor( companion object { // 项目视图管理 private const val PROJECT_VIEWS_MANAGER = "project_views_manager" + // 项目查看权限 private const val PROJECT_VIEW = "project_view" + // 项目访问权限 private const val PROJECT_VISIT = "project_visit" + // v3项目禁用启用 private const val PROJECT_DELETE = "project_delete" + // rbac项目禁用启用 private const val PROJECT_ENABLE = "project_enable" + // v3质量红线启用,rbac没有 private const val QUALITY_GROUP_ENABLE = "quality_group_enable" + // 流水线查看权限,v3没有pipeline_list权限,迁移至rbac需要添加 private const val PIPELINE_VIEW = "pipeline_view" + // 项目访问权限 private const val PIPELINE_LIST = "pipeline_list" @@ -383,7 +390,7 @@ class MigrateV3PolicyService constructor( } else { member.expiredAt } - groupService.addGroupMember( + permissionResourceMemberService.addGroupMember( userId = member.id, memberType = member.type, expiredAt = expiredAt, diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/sample/service/SampleAuthPermissionProjectService.kt b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/sample/service/SampleAuthPermissionProjectService.kt index d088ced6215..121f083af8d 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/sample/service/SampleAuthPermissionProjectService.kt +++ b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/sample/service/SampleAuthPermissionProjectService.kt @@ -31,6 +31,10 @@ class SampleAuthPermissionProjectService : PermissionProjectService { return true } + override fun checkUserInProjectLevelGroup(userId: String, projectCode: String): Boolean { + return false + } + override fun checkProjectManager(userId: String, projectCode: String): Boolean { return true } diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/sample/service/SamplePermissionApplyService.kt b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/sample/service/SamplePermissionApplyService.kt index f05d49fec8f..4b05dbc4dc9 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/sample/service/SamplePermissionApplyService.kt +++ b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/sample/service/SamplePermissionApplyService.kt @@ -33,7 +33,6 @@ import com.tencent.devops.auth.pojo.SearchGroupInfo import com.tencent.devops.auth.pojo.vo.ActionInfoVo import com.tencent.devops.auth.pojo.vo.AuthApplyRedirectInfoVo import com.tencent.devops.auth.pojo.vo.AuthRedirectGroupInfoVo -import com.tencent.devops.auth.pojo.vo.GroupPermissionDetailVo import com.tencent.devops.auth.pojo.vo.ManagerRoleGroupVO import com.tencent.devops.auth.pojo.vo.ResourceTypeInfoVo import com.tencent.devops.auth.service.iam.PermissionApplyService @@ -47,7 +46,11 @@ class SamplePermissionApplyService : PermissionApplyService { return emptyList() } - override fun listGroups(userId: String, projectId: String, searchGroupInfo: SearchGroupInfo): ManagerRoleGroupVO { + override fun listGroupsForApply( + userId: String, + projectId: String, + searchGroupInfo: SearchGroupInfo + ): ManagerRoleGroupVO { return ManagerRoleGroupVO(count = 0, results = emptyList()) } @@ -55,10 +58,6 @@ class SamplePermissionApplyService : PermissionApplyService { return true } - override fun getGroupPermissionDetail(userId: String, groupId: Int): Map> { - return emptyMap() - } - override fun getRedirectInformation( userId: String, projectId: String, diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/sample/service/SamplePermissionResourceGroupService.kt b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/sample/service/SamplePermissionResourceGroupService.kt index 85b0245b8b1..a0069dc9698 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/sample/service/SamplePermissionResourceGroupService.kt +++ b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/sample/service/SamplePermissionResourceGroupService.kt @@ -28,8 +28,9 @@ package com.tencent.devops.auth.provider.sample.service -import com.tencent.devops.auth.pojo.dto.GroupMemberRenewalDTO +import com.tencent.devops.auth.pojo.dto.GroupAddDTO import com.tencent.devops.auth.pojo.dto.RenameGroupDTO +import com.tencent.devops.auth.pojo.vo.GroupPermissionDetailVo import com.tencent.devops.auth.pojo.vo.IamGroupInfoVo import com.tencent.devops.auth.pojo.vo.IamGroupMemberInfoVo import com.tencent.devops.auth.pojo.vo.IamGroupPoliciesVo @@ -66,27 +67,8 @@ class SamplePermissionResourceGroupService : PermissionResourceGroupService { return emptyList() } - override fun renewal( - userId: String, - projectId: String, - resourceType: String, - groupId: Int, - memberRenewalDTO: GroupMemberRenewalDTO - ): Boolean { - return true - } - - override fun deleteGroupMember( - userId: String, - projectId: String, - resourceType: String, - groupId: Int - ): Boolean { - return true - } - override fun deleteGroup( - userId: String, + userId: String?, projectId: String, resourceType: String, groupId: Int @@ -94,16 +76,12 @@ class SamplePermissionResourceGroupService : PermissionResourceGroupService { return true } - override fun createGroupByGroupCode( - userId: String, + override fun createGroup( projectId: String, - resourceType: String, - groupCode: String - ): Boolean { - return true - } + groupAddDTO: GroupAddDTO + ): Int = 0 - override fun rename( + override fun renameGroup( userId: String, projectId: String, resourceType: String, @@ -113,13 +91,12 @@ class SamplePermissionResourceGroupService : PermissionResourceGroupService { return true } - override fun addGroupMember( - userId: String, - /*user 或 department*/ - memberType: String, - expiredAt: Long, - groupId: Int - ): Boolean { - return true + override fun getGroupPermissionDetail(groupId: Int): Map> { + return emptyMap() } + + override fun createProjectGroupByGroupCode( + projectId: String, + groupCode: String + ) = true } diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/sample/service/SamplePermissionResourceMemberService.kt b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/sample/service/SamplePermissionResourceMemberService.kt index 47a9b2182da..9210ca39780 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/sample/service/SamplePermissionResourceMemberService.kt +++ b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/sample/service/SamplePermissionResourceMemberService.kt @@ -1,5 +1,6 @@ package com.tencent.devops.auth.provider.sample.service +import com.tencent.devops.auth.pojo.dto.GroupMemberRenewalDTO import com.tencent.devops.auth.service.iam.PermissionResourceMemberService import com.tencent.devops.common.auth.api.pojo.BkAuthGroup import com.tencent.devops.common.auth.api.pojo.BkAuthGroupAndUserList @@ -23,7 +24,6 @@ class SamplePermissionResourceMemberService : PermissionResourceMemberService { } override fun batchAddResourceGroupMembers( - userId: String, projectCode: String, iamGroupId: Int, expiredTime: Long, @@ -32,7 +32,6 @@ class SamplePermissionResourceMemberService : PermissionResourceMemberService { ) = true override fun batchDeleteResourceGroupMembers( - userId: String, projectCode: String, iamGroupId: Int, members: List?, @@ -45,4 +44,26 @@ class SamplePermissionResourceMemberService : PermissionResourceMemberService { ): Int = 0 override fun autoRenewal(projectCode: String, resourceType: String, resourceCode: String) = Unit + + override fun renewalGroupMember( + userId: String, + projectCode: String, + resourceType: String, + groupId: Int, + memberRenewalDTO: GroupMemberRenewalDTO + ): Boolean = true + + override fun deleteGroupMember( + userId: String, + projectCode: String, + resourceType: String, + groupId: Int + ): Boolean = true + + override fun addGroupMember( + userId: String, + memberType: String, + expiredAt: Long, + groupId: Int + ): Boolean = true } diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/stream/service/CentralizedStramPermissionServiceImpl.kt b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/stream/service/CentralizedStramPermissionServiceImpl.kt index 0adda249404..7d3705549cd 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/stream/service/CentralizedStramPermissionServiceImpl.kt +++ b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/stream/service/CentralizedStramPermissionServiceImpl.kt @@ -29,12 +29,17 @@ package com.tencent.devops.auth.provider.stream.service import com.tencent.devops.auth.utils.GitTypeUtils import com.tencent.devops.common.auth.api.AuthPermission +import com.tencent.devops.common.auth.api.pojo.BkAuthGroup import com.tencent.devops.common.client.Client import org.springframework.beans.factory.annotation.Autowired class CentralizedStramPermissionServiceImpl @Autowired constructor( val client: Client ) : StreamPermissionServiceImpl() { + override fun getProjectUsers(projectCode: String, group: BkAuthGroup?): List { + return emptyList() + } + override fun isPublicProject(projectCode: String, userId: String?): Boolean { val gitType = GitTypeUtils.getType() // type: github, gitlab, svn, tgitd等 diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/stream/service/GithubStreamPermissionServiceImpl.kt b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/stream/service/GithubStreamPermissionServiceImpl.kt index ebac1e08273..06bdf7aece0 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/stream/service/GithubStreamPermissionServiceImpl.kt +++ b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/stream/service/GithubStreamPermissionServiceImpl.kt @@ -29,6 +29,7 @@ package com.tencent.devops.auth.provider.stream.service import com.google.common.cache.CacheBuilder import com.tencent.devops.common.auth.api.AuthPermission +import com.tencent.devops.common.auth.api.pojo.BkAuthGroup import com.tencent.devops.common.client.Client import com.tencent.devops.repository.api.github.ServiceGithubPermissionResource import com.tencent.devops.stream.api.service.ServiceStreamBasicSettingResource @@ -64,6 +65,10 @@ class GithubStreamPermissionServiceImpl @Autowired constructor( .expireAfterAccess(5, TimeUnit.MINUTES) .build() + override fun getProjectUsers(projectCode: String, group: BkAuthGroup?): List { + return emptyList() + } + override fun isPublicProject(projectCode: String, userId: String?): Boolean { if (publicProjectCache.getIfPresent(projectCode) != null) { return publicProjectCache.getIfPresent(projectCode)!! diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/stream/service/GitlabStreamPermissionServiceImpl.kt b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/stream/service/GitlabStreamPermissionServiceImpl.kt index 0e619dd21f8..e9858dd8ffc 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/stream/service/GitlabStreamPermissionServiceImpl.kt +++ b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/stream/service/GitlabStreamPermissionServiceImpl.kt @@ -28,9 +28,14 @@ package com.tencent.devops.auth.provider.stream.service import com.tencent.devops.common.auth.api.AuthPermission +import com.tencent.devops.common.auth.api.pojo.BkAuthGroup import org.springframework.beans.factory.annotation.Autowired class GitlabStreamPermissionServiceImpl @Autowired constructor() : StreamPermissionServiceImpl() { + override fun getProjectUsers(projectCode: String, group: BkAuthGroup?): List { + return emptyList() + } + override fun isPublicProject(projectCode: String, userId: String?): Boolean { TODO("Not yet implemented") } diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/stream/service/StreamPermissionProjectServiceImpl.kt b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/stream/service/StreamPermissionProjectServiceImpl.kt index 3f5edcbc449..696894dcd42 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/stream/service/StreamPermissionProjectServiceImpl.kt +++ b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/stream/service/StreamPermissionProjectServiceImpl.kt @@ -38,8 +38,7 @@ class StreamPermissionProjectServiceImpl @Autowired constructor( private val streamPermissionService: StreamPermissionServiceImpl ) : PermissionProjectService { override fun getProjectUsers(projectCode: String, group: BkAuthGroup?): List { - // stream场景下使用不到此接口。占做默认实现 - return emptyList() + return streamPermissionService.getProjectUsers(projectCode, group) } override fun getProjectGroupAndUserList(projectCode: String): List { @@ -68,6 +67,10 @@ class StreamPermissionProjectServiceImpl @Autowired constructor( return streamPermissionService.isProjectMember(projectCode, userId).first } + override fun checkUserInProjectLevelGroup(userId: String, projectCode: String): Boolean { + return true + } + override fun checkProjectManager( userId: String, projectCode: String diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/stream/service/StreamPermissionServiceImpl.kt b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/stream/service/StreamPermissionServiceImpl.kt index 96d50afb4a9..f6c34e9e292 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/stream/service/StreamPermissionServiceImpl.kt +++ b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/provider/stream/service/StreamPermissionServiceImpl.kt @@ -30,6 +30,7 @@ package com.tencent.devops.auth.provider.stream.service import com.tencent.devops.auth.service.iam.PermissionService import com.tencent.devops.common.auth.api.AuthPermission import com.tencent.devops.common.auth.api.pojo.AuthResourceInstance +import com.tencent.devops.common.auth.api.pojo.BkAuthGroup import com.tencent.devops.common.auth.utils.ActionTypeUtils import org.slf4j.LoggerFactory @@ -191,6 +192,8 @@ abstract class StreamPermissionServiceImpl : PermissionService { return emptyMap() } + abstract fun getProjectUsers(projectCode: String, group: BkAuthGroup?): List + /** * 是否是开源项目 * projectCode: stream侧项目编码 diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/UserAuthApplyResourceImpl.kt b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/UserAuthApplyResourceImpl.kt index d5dcdc1de80..67eadcae511 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/UserAuthApplyResourceImpl.kt +++ b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/UserAuthApplyResourceImpl.kt @@ -9,6 +9,7 @@ import com.tencent.devops.auth.pojo.vo.GroupPermissionDetailVo import com.tencent.devops.auth.pojo.vo.ManagerRoleGroupVO import com.tencent.devops.auth.pojo.vo.ResourceTypeInfoVo import com.tencent.devops.auth.service.iam.PermissionApplyService +import com.tencent.devops.auth.service.iam.PermissionResourceGroupService import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.auth.api.AuthResourceType import com.tencent.devops.common.web.RestResource @@ -16,7 +17,8 @@ import org.springframework.beans.factory.annotation.Autowired @RestResource class UserAuthApplyResourceImpl @Autowired constructor( - val permissionApplyService: PermissionApplyService + val permissionApplyService: PermissionApplyService, + val permissionResourceGroupService: PermissionResourceGroupService ) : UserAuthApplyResource { override fun listResourceTypes(userId: String): Result> { return Result( @@ -40,7 +42,7 @@ class UserAuthApplyResourceImpl @Autowired constructor( searchGroupInfo: SearchGroupInfo ): Result { return Result( - permissionApplyService.listGroups( + permissionApplyService.listGroupsForApply( userId = userId, projectId = projectId, searchGroupInfo = searchGroupInfo @@ -62,8 +64,7 @@ class UserAuthApplyResourceImpl @Autowired constructor( groupId: Int ): Result>> { return Result( - permissionApplyService.getGroupPermissionDetail( - userId = userId, + permissionResourceGroupService.getGroupPermissionDetail( groupId = groupId ) ) diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/UserAuthResourceGroupResourceImpl.kt b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/UserAuthResourceGroupResourceImpl.kt index f8e576da207..1b56fa3fef1 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/UserAuthResourceGroupResourceImpl.kt +++ b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/UserAuthResourceGroupResourceImpl.kt @@ -33,13 +33,15 @@ import com.tencent.devops.auth.pojo.dto.GroupMemberRenewalDTO import com.tencent.devops.auth.pojo.dto.RenameGroupDTO import com.tencent.devops.auth.pojo.vo.IamGroupPoliciesVo import com.tencent.devops.auth.service.iam.PermissionResourceGroupService +import com.tencent.devops.auth.service.iam.PermissionResourceMemberService import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.web.RestResource import org.springframework.beans.factory.annotation.Autowired @RestResource class UserAuthResourceGroupResourceImpl @Autowired constructor( - private val permissionResourceGroupService: PermissionResourceGroupService + private val permissionResourceGroupService: PermissionResourceGroupService, + private val permissionResourceMemberService: PermissionResourceMemberService ) : UserAuthResourceGroupResource { override fun getGroupPolicies( userId: String, @@ -65,9 +67,9 @@ class UserAuthResourceGroupResourceImpl @Autowired constructor( memberRenewalDTO: GroupMemberRenewalDTO ): Result { return Result( - permissionResourceGroupService.renewal( + permissionResourceMemberService.renewalGroupMember( userId = userId, - projectId = projectId, + projectCode = projectId, resourceType = resourceType, groupId = groupId, memberRenewalDTO = memberRenewalDTO @@ -82,9 +84,9 @@ class UserAuthResourceGroupResourceImpl @Autowired constructor( groupId: Int ): Result { return Result( - permissionResourceGroupService.deleteGroupMember( + permissionResourceMemberService.deleteGroupMember( userId = userId, - projectId = projectId, + projectCode = projectId, resourceType = resourceType, groupId = groupId ) @@ -115,7 +117,7 @@ class UserAuthResourceGroupResourceImpl @Autowired constructor( renameGroupDTO: RenameGroupDTO ): Result { return Result( - permissionResourceGroupService.rename( + permissionResourceGroupService.renameGroup( userId = userId, projectId = projectId, resourceType = resourceType, diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/service/ServiceManagerResourceImpl.kt b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/service/ServiceManagerResourceImpl.kt index 93bd32eae7e..9ecb56fb985 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/service/ServiceManagerResourceImpl.kt +++ b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/service/ServiceManagerResourceImpl.kt @@ -31,12 +31,15 @@ import com.tencent.devops.auth.api.service.ServiceManagerResource import com.tencent.devops.auth.service.SuperManagerService import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.web.RestResource +import com.tencent.devops.common.web.annotation.BkApiPermission +import com.tencent.devops.common.web.constant.BkApiHandleType import org.springframework.beans.factory.annotation.Autowired @RestResource class ServiceManagerResourceImpl @Autowired constructor( val superManagerService: SuperManagerService ) : ServiceManagerResource { + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun validateManagerPermission( userId: String, token: String, diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/service/ServicePermissionAuthResourceImpl.kt b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/service/ServicePermissionAuthResourceImpl.kt index 977320d05ad..0772fa5cfb7 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/service/ServicePermissionAuthResourceImpl.kt +++ b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/service/ServicePermissionAuthResourceImpl.kt @@ -36,6 +36,8 @@ import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.auth.api.AuthPermission import com.tencent.devops.common.auth.api.pojo.AuthResourceInstance import com.tencent.devops.common.web.RestResource +import com.tencent.devops.common.web.annotation.BkApiPermission +import com.tencent.devops.common.web.constant.BkApiHandleType import org.springframework.beans.factory.annotation.Autowired @RestResource @@ -45,6 +47,7 @@ class ServicePermissionAuthResourceImpl @Autowired constructor( val permissionGrantService: PermissionGrantService ) : ServicePermissionAuthResource { + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun validateUserActionPermission( userId: String, token: String, @@ -54,6 +57,7 @@ class ServicePermissionAuthResourceImpl @Autowired constructor( return Result(permissionService.validateUserActionPermission(userId, action)) } + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun validateUserResourcePermission( userId: String, token: String, @@ -65,6 +69,7 @@ class ServicePermissionAuthResourceImpl @Autowired constructor( return Result(permissionService.validateUserResourcePermission(userId, action, projectCode, resourceCode)) } + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun validateUserResourcePermissionByRelation( userId: String, token: String, @@ -87,6 +92,7 @@ class ServicePermissionAuthResourceImpl @Autowired constructor( ) } + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun validateUserResourcePermissionByInstance( userId: String, token: String, @@ -105,6 +111,7 @@ class ServicePermissionAuthResourceImpl @Autowired constructor( ) } + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun batchValidateUserResourcePermissionByRelation( userId: String, token: String, @@ -133,6 +140,7 @@ class ServicePermissionAuthResourceImpl @Autowired constructor( return Result(actionCheckPermission) } + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun getUserResourceByPermission( userId: String, token: String, @@ -151,6 +159,7 @@ class ServicePermissionAuthResourceImpl @Autowired constructor( ) } + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun getUserResourcesByPermissions( userId: String, token: String, @@ -169,6 +178,7 @@ class ServicePermissionAuthResourceImpl @Autowired constructor( ) } + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun filterUserResourcesByPermissions( userId: String, token: String, @@ -189,6 +199,7 @@ class ServicePermissionAuthResourceImpl @Autowired constructor( ) } + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun getUserResourceAndParentByPermission( userId: String, token: String, @@ -207,6 +218,7 @@ class ServicePermissionAuthResourceImpl @Autowired constructor( ) } + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun resourceCreateRelation( userId: String, token: String, @@ -227,6 +239,7 @@ class ServicePermissionAuthResourceImpl @Autowired constructor( ) } + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun resourceModifyRelation( token: String, type: String?, @@ -245,6 +258,7 @@ class ServicePermissionAuthResourceImpl @Autowired constructor( ) } + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun resourceDeleteRelation( token: String, type: String?, @@ -261,6 +275,7 @@ class ServicePermissionAuthResourceImpl @Autowired constructor( ) } + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun resourceCancelRelation( userId: String, token: String, @@ -279,6 +294,7 @@ class ServicePermissionAuthResourceImpl @Autowired constructor( ) } + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun grantInstancePermission( userId: String, token: String, diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/service/ServiceProjectAuthResourceImpl.kt b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/service/ServiceProjectAuthResourceImpl.kt index 427a6ac16b2..e94939ddc5b 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/service/ServiceProjectAuthResourceImpl.kt +++ b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/service/ServiceProjectAuthResourceImpl.kt @@ -35,12 +35,15 @@ import com.tencent.devops.common.auth.api.pojo.BKAuthProjectRolesResources import com.tencent.devops.common.auth.api.pojo.BkAuthGroup import com.tencent.devops.common.auth.api.pojo.BkAuthGroupAndUserList import com.tencent.devops.common.web.RestResource +import com.tencent.devops.common.web.annotation.BkApiPermission +import com.tencent.devops.common.web.constant.BkApiHandleType import org.springframework.beans.factory.annotation.Autowired @RestResource class ServiceProjectAuthResourceImpl @Autowired constructor( val permissionProjectService: PermissionProjectService ) : ServiceProjectAuthResource { + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun getProjectUsers( token: String, type: String?, @@ -55,6 +58,7 @@ class ServiceProjectAuthResourceImpl @Autowired constructor( ) } + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun getProjectGroupAndUserList( token: String, projectCode: String @@ -64,10 +68,12 @@ class ServiceProjectAuthResourceImpl @Autowired constructor( ) } + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun getUserProjects(token: String, userId: String): Result> { return Result(permissionProjectService.getUserProjects(userId)) } + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun getUserProjectsByPermission( token: String, userId: String, @@ -83,6 +89,7 @@ class ServiceProjectAuthResourceImpl @Autowired constructor( ) } + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun isProjectUser( token: String, type: String?, @@ -99,12 +106,28 @@ class ServiceProjectAuthResourceImpl @Autowired constructor( ) } + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) + override fun checkUserInProjectLevelGroup( + token: String, + userId: String, + projectCode: String + ): Result { + return Result( + permissionProjectService.checkUserInProjectLevelGroup( + userId = userId, + projectCode = projectCode + ) + ) + } + + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun checkManager(token: String, userId: String, projectId: String): Result { val result = permissionProjectService.checkProjectManager(userId, projectId) || permissionProjectService.isProjectUser(userId, projectId, BkAuthGroup.CI_MANAGER) return Result(result) } + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun checkProjectManager( token: String, type: String?, @@ -119,6 +142,7 @@ class ServiceProjectAuthResourceImpl @Autowired constructor( ) } + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun createProjectUser( token: String, userId: String, @@ -134,6 +158,7 @@ class ServiceProjectAuthResourceImpl @Autowired constructor( ) } + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun batchCreateProjectUser( token: String, userId: String, @@ -151,6 +176,7 @@ class ServiceProjectAuthResourceImpl @Autowired constructor( ) } + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun getProjectRoles( token: String, projectCode: String, @@ -164,6 +190,7 @@ class ServiceProjectAuthResourceImpl @Autowired constructor( ) } + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun getProjectPermissionInfo( token: String, projectCode: String diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/service/ServiceResourceGroupResourceImpl.kt b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/service/ServiceResourceGroupResourceImpl.kt index 1a94a2b0bb4..9553befeb08 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/service/ServiceResourceGroupResourceImpl.kt +++ b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/service/ServiceResourceGroupResourceImpl.kt @@ -1,7 +1,8 @@ package com.tencent.devops.auth.resources.service import com.tencent.devops.auth.api.service.ServiceResourceGroupResource -import com.tencent.devops.auth.service.iam.PermissionProjectService +import com.tencent.devops.auth.pojo.dto.GroupAddDTO +import com.tencent.devops.auth.pojo.vo.GroupPermissionDetailVo import com.tencent.devops.auth.service.iam.PermissionResourceGroupService import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.auth.api.pojo.BkAuthGroup @@ -9,34 +10,52 @@ import com.tencent.devops.common.web.RestResource @RestResource class ServiceResourceGroupResourceImpl constructor( - val permissionResourceGroupService: PermissionResourceGroupService, - val permissionProjectService: PermissionProjectService + val permissionResourceGroupService: PermissionResourceGroupService ) : ServiceResourceGroupResource { + override fun getGroupPermissionDetail( + projectCode: String, + groupId: Int + ): Result>> { + return Result( + permissionResourceGroupService.getGroupPermissionDetail( + groupId = groupId + ) + ) + } + override fun createGroupByGroupCode( - userId: String, projectCode: String, resourceType: String, groupCode: BkAuthGroup ): Result { return Result( - permissionResourceGroupService.createGroupByGroupCode( - userId = userId, + permissionResourceGroupService.createProjectGroupByGroupCode( projectId = projectCode, - groupCode = groupCode.value, - resourceType = resourceType + groupCode = groupCode.value + ) + ) + } + + override fun createGroup( + projectCode: String, + groupAddDTO: GroupAddDTO + ): Result { + return Result( + data = permissionResourceGroupService.createGroup( + projectId = projectCode, + groupAddDTO = groupAddDTO ) ) } override fun deleteGroup( - userId: String, projectCode: String, resourceType: String, groupId: Int ): Result { return Result( permissionResourceGroupService.deleteGroup( - userId = userId, + userId = null, projectId = projectCode, groupId = groupId, resourceType = resourceType diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/service/ServiceResourceMemberResourceImpl.kt b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/service/ServiceResourceMemberResourceImpl.kt index 77825d42d0a..c9946730b02 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/service/ServiceResourceMemberResourceImpl.kt +++ b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/resources/service/ServiceResourceMemberResourceImpl.kt @@ -1,23 +1,22 @@ package com.tencent.devops.auth.resources.service import com.tencent.devops.auth.api.service.ServiceResourceMemberResource -import com.tencent.devops.auth.service.iam.PermissionProjectService import com.tencent.devops.auth.service.iam.PermissionResourceMemberService -import com.tencent.devops.common.api.exception.ErrorCodeException import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.auth.api.pojo.BkAuthGroup import com.tencent.devops.common.auth.api.pojo.BkAuthGroupAndUserList import com.tencent.devops.common.web.RestResource -import com.tencent.devops.project.constant.ProjectMessageCode +import com.tencent.devops.common.web.annotation.BkApiPermission +import com.tencent.devops.common.web.constant.BkApiHandleType import com.tencent.devops.project.pojo.ProjectCreateUserInfo import com.tencent.devops.project.pojo.ProjectDeleteUserInfo import java.util.concurrent.TimeUnit @RestResource class ServiceResourceMemberResourceImpl constructor( - private val permissionResourceMemberService: PermissionResourceMemberService, - private val permissionProjectService: PermissionProjectService + private val permissionResourceMemberService: PermissionResourceMemberService ) : ServiceResourceMemberResource { + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun getResourceGroupMembers( token: String, projectCode: String, @@ -35,6 +34,7 @@ class ServiceResourceMemberResourceImpl constructor( ) } + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun getResourceGroupAndMembers( token: String, projectCode: String, @@ -50,21 +50,16 @@ class ServiceResourceMemberResourceImpl constructor( ) } + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun batchAddResourceGroupMembers( token: String, - userId: String, projectCode: String, projectCreateUserInfo: ProjectCreateUserInfo ): Result { - verifyProjectManager( - userId = userId, - projectCode = projectCode - ) with(projectCreateUserInfo) { val expiredTime = System.currentTimeMillis() / 1000 + TimeUnit.DAYS.toSeconds(365L) return Result( permissionResourceMemberService.batchAddResourceGroupMembers( - userId = userId, projectCode = projectCode, iamGroupId = getIamGroupId( groupId = groupId, @@ -80,20 +75,15 @@ class ServiceResourceMemberResourceImpl constructor( } } + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun batchDeleteResourceGroupMembers( token: String, - userId: String, projectCode: String, projectDeleteUserInfo: ProjectDeleteUserInfo ): Result { - verifyProjectManager( - userId = userId, - projectCode = projectCode - ) with(projectDeleteUserInfo) { return Result( permissionResourceMemberService.batchDeleteResourceGroupMembers( - userId = userId, projectCode = projectCode, iamGroupId = getIamGroupId( groupId = groupId, @@ -108,22 +98,6 @@ class ServiceResourceMemberResourceImpl constructor( } } - private fun verifyProjectManager( - userId: String, - projectCode: String - ) { - val isProjectManager = permissionProjectService.checkProjectManager( - userId = userId, - projectCode = projectCode - ) - if (!isProjectManager) { - throw ErrorCodeException( - errorCode = ProjectMessageCode.NOT_MANAGER, - defaultMessage = "The user($userId) is not the manager of the project($projectCode)!" - ) - } - } - private fun getIamGroupId( groupId: Int?, projectCode: String, diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/service/iam/PermissionApplyService.kt b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/service/iam/PermissionApplyService.kt index 70cedd212c1..b9cfdd28227 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/service/iam/PermissionApplyService.kt +++ b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/service/iam/PermissionApplyService.kt @@ -4,7 +4,6 @@ import com.tencent.devops.auth.pojo.ApplyJoinGroupInfo import com.tencent.devops.auth.pojo.SearchGroupInfo import com.tencent.devops.auth.pojo.vo.ActionInfoVo import com.tencent.devops.auth.pojo.vo.AuthApplyRedirectInfoVo -import com.tencent.devops.auth.pojo.vo.GroupPermissionDetailVo import com.tencent.devops.auth.pojo.vo.ManagerRoleGroupVO import com.tencent.devops.auth.pojo.vo.ResourceTypeInfoVo @@ -16,7 +15,7 @@ interface PermissionApplyService { resourceType: String ): List - fun listGroups( + fun listGroupsForApply( userId: String, projectId: String, searchGroupInfo: SearchGroupInfo @@ -27,11 +26,6 @@ interface PermissionApplyService { applyJoinGroupInfo: ApplyJoinGroupInfo ): Boolean - fun getGroupPermissionDetail( - userId: String, - groupId: Int - ): Map> - fun getRedirectInformation( userId: String, projectId: String, diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/service/iam/PermissionProjectService.kt b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/service/iam/PermissionProjectService.kt index 0708e3a74ec..e40074a0715 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/service/iam/PermissionProjectService.kt +++ b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/service/iam/PermissionProjectService.kt @@ -48,6 +48,11 @@ interface PermissionProjectService { fun isProjectUser(userId: String, projectCode: String, group: BkAuthGroup?): Boolean + fun checkUserInProjectLevelGroup( + userId: String, + projectCode: String + ): Boolean + fun checkProjectManager(userId: String, projectCode: String): Boolean fun createProjectUser(userId: String, projectCode: String, roleCode: String): Boolean diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/service/iam/PermissionResourceGroupService.kt b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/service/iam/PermissionResourceGroupService.kt index 49f4b54e25b..58912f914ba 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/service/iam/PermissionResourceGroupService.kt +++ b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/service/iam/PermissionResourceGroupService.kt @@ -28,8 +28,9 @@ package com.tencent.devops.auth.service.iam -import com.tencent.devops.auth.pojo.dto.GroupMemberRenewalDTO +import com.tencent.devops.auth.pojo.dto.GroupAddDTO import com.tencent.devops.auth.pojo.dto.RenameGroupDTO +import com.tencent.devops.auth.pojo.vo.GroupPermissionDetailVo import com.tencent.devops.auth.pojo.vo.IamGroupInfoVo import com.tencent.devops.auth.pojo.vo.IamGroupMemberInfoVo import com.tencent.devops.auth.pojo.vo.IamGroupPoliciesVo @@ -67,39 +68,19 @@ interface PermissionResourceGroupService { groupId: Int ): List - /** - * 用户续期 - */ - fun renewal( - userId: String, - projectId: String, - resourceType: String, - groupId: Int, - memberRenewalDTO: GroupMemberRenewalDTO - ): Boolean - - fun deleteGroupMember( - userId: String, - projectId: String, - resourceType: String, - groupId: Int - ): Boolean - fun deleteGroup( - userId: String, + userId: String?, projectId: String, resourceType: String, groupId: Int ): Boolean - fun createGroupByGroupCode( - userId: String, + fun createGroup( projectId: String, - resourceType: String, - groupCode: String - ): Boolean + groupAddDTO: GroupAddDTO + ): Int - fun rename( + fun renameGroup( userId: String, projectId: String, resourceType: String, @@ -107,11 +88,10 @@ interface PermissionResourceGroupService { renameGroupDTO: RenameGroupDTO ): Boolean - fun addGroupMember( - userId: String, - /*user or department or template*/ - memberType: String, - expiredAt: Long, - groupId: Int + fun getGroupPermissionDetail(groupId: Int): Map> + + fun createProjectGroupByGroupCode( + projectId: String, + groupCode: String ): Boolean } diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/service/iam/PermissionResourceMemberService.kt b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/service/iam/PermissionResourceMemberService.kt index 75a4f0ba97a..fe4aa8fe4cc 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/service/iam/PermissionResourceMemberService.kt +++ b/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/service/iam/PermissionResourceMemberService.kt @@ -1,5 +1,6 @@ package com.tencent.devops.auth.service.iam +import com.tencent.devops.auth.pojo.dto.GroupMemberRenewalDTO import com.tencent.devops.common.auth.api.pojo.BkAuthGroup import com.tencent.devops.common.auth.api.pojo.BkAuthGroupAndUserList @@ -19,7 +20,6 @@ interface PermissionResourceMemberService { @Suppress("LongParameterList") fun batchAddResourceGroupMembers( - userId: String, projectCode: String, iamGroupId: Int, expiredTime: Long, @@ -28,7 +28,6 @@ interface PermissionResourceMemberService { ): Boolean fun batchDeleteResourceGroupMembers( - userId: String, projectCode: String, iamGroupId: Int, members: List? = emptyList(), @@ -45,4 +44,27 @@ interface PermissionResourceMemberService { resourceType: String, resourceCode: String ) + + fun renewalGroupMember( + userId: String, + projectCode: String, + resourceType: String, + groupId: Int, + memberRenewalDTO: GroupMemberRenewalDTO + ): Boolean + + fun deleteGroupMember( + userId: String, + projectCode: String, + resourceType: String, + groupId: Int + ): Boolean + + fun addGroupMember( + userId: String, + /*user or department or template*/ + memberType: String, + expiredAt: Long, + groupId: Int + ): Boolean } diff --git a/src/backend/ci/core/auth/biz-auth/src/test/kotlin/com/tencent/devops/auth/rbac/service/migrate/AbMigratePolicyServiceTest.kt b/src/backend/ci/core/auth/biz-auth/src/test/kotlin/com/tencent/devops/auth/rbac/service/migrate/AbMigratePolicyServiceTest.kt index ac79556aefa..0a1f6ea6095 100644 --- a/src/backend/ci/core/auth/biz-auth/src/test/kotlin/com/tencent/devops/auth/rbac/service/migrate/AbMigratePolicyServiceTest.kt +++ b/src/backend/ci/core/auth/biz-auth/src/test/kotlin/com/tencent/devops/auth/rbac/service/migrate/AbMigratePolicyServiceTest.kt @@ -44,6 +44,7 @@ import com.tencent.devops.auth.provider.rbac.service.migrate.MigrateIamApiServic import com.tencent.devops.auth.provider.rbac.service.migrate.MigrateResourceCodeConverter import com.tencent.devops.auth.service.DeptService import com.tencent.devops.auth.service.iam.PermissionResourceGroupService +import com.tencent.devops.auth.service.iam.PermissionResourceMemberService import com.tencent.devops.auth.service.iam.PermissionService import com.tencent.devops.common.api.util.JsonUtil import com.tencent.devops.common.test.BkCiAbstractTest @@ -68,6 +69,7 @@ open class AbMigratePolicyServiceTest : BkCiAbstractTest() { val deptService: DeptService = mockk() val permissionGroupPoliciesService: PermissionGroupPoliciesService = mockk() val groupService: PermissionResourceGroupService = mockk() + val permissionResourceMemberService: PermissionResourceMemberService = mockk() @BeforeEach fun before() { diff --git a/src/backend/ci/core/auth/biz-auth/src/test/kotlin/com/tencent/devops/auth/rbac/service/migrate/MigrateV0PolicyServiceTest.kt b/src/backend/ci/core/auth/biz-auth/src/test/kotlin/com/tencent/devops/auth/rbac/service/migrate/MigrateV0PolicyServiceTest.kt index c0abfc1a3c9..52976010208 100644 --- a/src/backend/ci/core/auth/biz-auth/src/test/kotlin/com/tencent/devops/auth/rbac/service/migrate/MigrateV0PolicyServiceTest.kt +++ b/src/backend/ci/core/auth/biz-auth/src/test/kotlin/com/tencent/devops/auth/rbac/service/migrate/MigrateV0PolicyServiceTest.kt @@ -60,7 +60,7 @@ class MigrateV0PolicyServiceTest : AbMigratePolicyServiceTest() { authMigrationDao = authMigrationDao, deptService = deptService, permissionGroupPoliciesService = permissionGroupPoliciesService, - groupService = groupService + permissionResourceMemberService = permissionResourceMemberService ), recordPrivateCalls = true ) diff --git a/src/backend/ci/core/auth/biz-auth/src/test/kotlin/com/tencent/devops/auth/rbac/service/migrate/MigrateV3PolicyServiceTest.kt b/src/backend/ci/core/auth/biz-auth/src/test/kotlin/com/tencent/devops/auth/rbac/service/migrate/MigrateV3PolicyServiceTest.kt index cbff4d9ac89..403007b90a8 100644 --- a/src/backend/ci/core/auth/biz-auth/src/test/kotlin/com/tencent/devops/auth/rbac/service/migrate/MigrateV3PolicyServiceTest.kt +++ b/src/backend/ci/core/auth/biz-auth/src/test/kotlin/com/tencent/devops/auth/rbac/service/migrate/MigrateV3PolicyServiceTest.kt @@ -59,7 +59,7 @@ class MigrateV3PolicyServiceTest : AbMigratePolicyServiceTest() { authMigrationDao = authMigrationDao, deptService = deptService, permissionGroupPoliciesService = permissionGroupPoliciesService, - groupService = groupService + permissionResourceMemberService = permissionResourceMemberService ), recordPrivateCalls = true ) diff --git a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/auth/Header.kt b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/auth/Header.kt index ebfe09ed94e..d53a1e0b28f 100644 --- a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/auth/Header.kt +++ b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/auth/Header.kt @@ -66,6 +66,7 @@ const val AUTH_HEADER_DEVOPS_PIPELINE_ID: String = "X-DEVOPS-PIPELINE-ID" const val AUTH_HEADER_DEVOPS_BUILD_ID: String = "X-DEVOPS-BUILD-ID" const val AUTH_HEADER_DEVOPS_VM_SEQ_ID: String = "X-DEVOPS-VM-SID" const val AUTH_HEADER_DEVOPS_VM_NAME: String = "X-DEVOPS-VM-NAME" +const val AUTH_HEADER_DEVOPS_EXECUTE_COUNT: String = "X-DEVOPS-EXECUTE-COUNT" const val AUTH_HEADER_DEVOPS_REAL_IP: String = "X-DEVOPS-REAL-IP" @@ -111,3 +112,10 @@ const val REFERER = "referer" // 来源 const val DEVX_HEADER_GW_TOKEN = "X-DEVOPS-DEVX-GW-TOKEN" const val DEVX_HEADER_NGGW_CLIENT_ADDRESS = "X-BK-NGGW-CLIENT-ADDRESS" + +const val AUTH_HEADER_DEVOPS_SHA_CONTENT: String = "X-DEVOPS-SHA-CONTENT" +const val AUTH_HEADER_DEVOPS_OS_NAME: String = "X-DEVOPS-OS-NAME" +const val AUTH_HEADER_DEVOPS_OS_ARCH: String = "X-DEVOPS-OS-ARCH" +const val AUTH_HEADER_DEVOPS_STORE_CODE: String = "X-DEVOPS-STORE-CODE" +const val AUTH_HEADER_DEVOPS_STORE_TYPE: String = "X-DEVOPS-STORE-TYPE" +const val AUTH_HEADER_DEVOPS_STORE_VERSION: String = "X-DEVOPS-STORE-VERSION" diff --git a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/constant/CommonConstants.kt b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/constant/CommonConstants.kt index 3db8f5324f0..a62190338dd 100644 --- a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/constant/CommonConstants.kt +++ b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/constant/CommonConstants.kt @@ -162,6 +162,8 @@ const val KEY_ARCHIVE = "archive" const val KEY_BRANCH_TEST_FLAG = "branchTestFlag" const val KEY_TASK_ATOM = "taskAtom" const val KEY_ELEMENT_ENABLE = "elementEnable" +const val KEY_SHA_CONTENT = "shaContent" +const val KEY_INSTALLED_PKG_SHA_CONTENT = "installedPkgShaContent" const val BK_BUILD_ENV_START_FAILED = "bkBuildEnvStartFailed" // 构建环境启动失败 const val BK_START_PULL_IMAGE = "bkStartPullImage" // 开始拉取镜像，镜像名称： diff --git a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/constant/CommonMessageCode.kt b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/constant/CommonMessageCode.kt index 9a5ce748d76..58a88d007b6 100644 --- a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/constant/CommonMessageCode.kt +++ b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/constant/CommonMessageCode.kt @@ -101,8 +101,6 @@ object CommonMessageCode { const val USERS_EXCEEDS_THE_LIMIT = "2100048" // 授权用户数越界:{0} const val FAILED_TO_QUERY_GSE_AGENT_STATUS = "2100049" // 查询 Gse Agent 状态失败 const val FAILED_TO_GET_AGENT_STATUS = "2100050" // 获取agent状态失败 - const val FAILED_TO_GET_CMDB_NODE = "2100051" // 获取 CMDB 节点失败 - const val FAILED_TO_GET_CMDB_LIST = "2100052" // 获取CMDB列表失败 const val STAGES_AND_STEPS_CANNOT_EXIST_BY_SIDE = "2100053" // stages和steps不能并列存在! const val USER_NOT_PERMISSIONS_OPERATE_PIPELINE = "2100054" // 用户({0})无权限在工程({1})下{2}流水线{3} @@ -183,6 +181,7 @@ object CommonMessageCode { const val YAML_NOT_VALID = "2100130" // yaml不合法 {0} const val GIT_INVALID_PRIVATE_KEY = "2100131" // 不支持的SSH私钥格式，仅支持rsa格式私钥 const val GIT_INVALID_PRIVATE_KEY_OR_PASSWORD = "2100132" // 第三方服务[{0}]操作失败，失败详情：{1} + const val MR_ACCEPT_EVENT_NOT_SUPPORT_TRANSFER = "2100133" // mr accept事件类型不支持code转换 const val SVN_TOKEN_FAIL = "2100135" // SVN Token 不正确 const val SVN_TOKEN_EMPTY = "2100136" // SVN Token 为空, 请检查代码库的凭证类型 diff --git a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/enums/TriggerRepositoryType.kt b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/enums/TriggerRepositoryType.kt index 6fd5e4093c5..bd1b2591819 100644 --- a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/enums/TriggerRepositoryType.kt +++ b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/enums/TriggerRepositoryType.kt @@ -30,7 +30,9 @@ package com.tencent.devops.common.api.enums enum class TriggerRepositoryType { ID, NAME, - SELF; + SELF, + // 触发器不需要绑定代码库,如定时触发默认时,不需要绑定 + NONE; companion object { fun toRepositoryType(type: TriggerRepositoryType?): RepositoryType? { diff --git a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/util/JsonUtil.kt b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/util/JsonUtil.kt index a6d020edc60..22828875ffa 100644 --- a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/util/JsonUtil.kt +++ b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/util/JsonUtil.kt @@ -32,9 +32,11 @@ import com.fasterxml.jackson.annotation.JsonInclude import com.fasterxml.jackson.core.json.JsonReadFeature import com.fasterxml.jackson.core.type.TypeReference import com.fasterxml.jackson.databind.DeserializationFeature +import com.fasterxml.jackson.databind.MapperFeature import com.fasterxml.jackson.databind.Module import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.SerializationFeature +import com.fasterxml.jackson.databind.json.JsonMapper import com.fasterxml.jackson.databind.ser.FilterProvider import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider @@ -119,22 +121,40 @@ object JsonUtil { private val objectMapper = objectMapper() + private val jsonMapper = jsonMapper() + private fun objectMapper(): ObjectMapper { return ObjectMapper().apply { - registerModule(javaTimeModule()) - registerModule(KotlinModule()) - enable(SerializationFeature.INDENT_OUTPUT) - enable(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT) - enable(JsonReadFeature.ALLOW_UNESCAPED_CONTROL_CHARS.mappedFeature()) - setSerializationInclusion(JsonInclude.Include.NON_NULL) - disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) - disable(SerializationFeature.FAIL_ON_EMPTY_BEANS) - jsonModules.forEach { jsonModule -> - registerModule(jsonModule) - } + objectMapperInit() + } + } + + private fun ObjectMapper.objectMapperInit() { + registerModule(javaTimeModule()) + registerModule(KotlinModule()) + enable(SerializationFeature.INDENT_OUTPUT) + enable(DeserializationFeature.ACCEPT_EMPTY_STRING_AS_NULL_OBJECT) + enable(JsonReadFeature.ALLOW_UNESCAPED_CONTROL_CHARS.mappedFeature()) + setSerializationInclusion(JsonInclude.Include.NON_NULL) + disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) + disable(SerializationFeature.FAIL_ON_EMPTY_BEANS) + jsonModules.forEach { jsonModule -> + registerModule(jsonModule) } } + private fun jsonMapper(): JsonMapper { + return JsonMapper.builder() + /* 使得POJO反序列化有序，对性能会有略微影响 + * https://github.com/FasterXML/jackson-databind/issues/3900 + * */ + .enable(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY) + .enable(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS) + .disable(MapperFeature.SORT_CREATOR_PROPERTIES_FIRST).build().apply { + objectMapperInit() + } + } + private val skipEmptyObjectMapper = ObjectMapper().apply { registerModule(javaTimeModule()) registerModule(KotlinModule()) @@ -178,6 +198,7 @@ object JsonUtil { } subModules.forEach { subModule -> objectMapper.registerModule(subModule) + jsonMapper.registerModule(subModule) skipEmptyObjectMapper.registerModule(subModule) unformattedObjectMapper.registerModule(subModule) } @@ -193,6 +214,13 @@ object JsonUtil { return getObjectMapper(formatted).writeValueAsString(bean)!! } + fun toSortJson(bean: Any): String { + if (ReflectUtil.isNativeType(bean) || bean is String) { + return bean.toString() + } + return jsonMapper.writeValueAsString(bean)!! + } + /** * 将对象转可修改的Map, * 注意：会忽略掉值为空串和null的属性 diff --git a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/util/OkhttpUtils.kt b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/util/OkhttpUtils.kt index c07f7257afb..f72f7dbabed 100644 --- a/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/util/OkhttpUtils.kt +++ b/src/backend/ci/core/common/common-api/src/main/kotlin/com/tencent/devops/common/api/util/OkhttpUtils.kt @@ -155,6 +155,10 @@ object OkhttpUtils { return doHttp(okHttpClient, request) } + fun doShortGet(url: String, headers: Map = mapOf()): Response { + return doGet(shortOkHttpClient, url, headers) + } + fun doLongGet(url: String, headers: Map = mapOf()): Response { return doGet(longHttpClient, url, headers) } @@ -167,6 +171,13 @@ object OkhttpUtils { return doHttp(shortOkHttpClient, request) } + fun doShortPost(url: String, jsonParam: String, headers: Map = mapOf()): Response { + val builder = getBuilder(url, headers) + val body = jsonParam.toRequestBody(jsonMediaType) + val request = builder.post(body).build() + return doShortHttp(request) + } + private fun doCustomClientHttp(customOkHttpClient: OkHttpClient, request: Request): Response { return doHttp(customOkHttpClient, request) } diff --git a/src/backend/ci/core/common/common-audit/src/main/kotlin/com/tencent/devops/common/audit/ActionAuditContent.kt b/src/backend/ci/core/common/common-audit/src/main/kotlin/com/tencent/devops/common/audit/ActionAuditContent.kt index 2658b4bd06f..f4fcc475290 100644 --- a/src/backend/ci/core/common/common-audit/src/main/kotlin/com/tencent/devops/common/audit/ActionAuditContent.kt +++ b/src/backend/ci/core/common/common-audit/src/main/kotlin/com/tencent/devops/common/audit/ActionAuditContent.kt @@ -94,6 +94,7 @@ object ActionAuditContent { const val CGS_DELETE_CONTENT = "delete workspace $CONTENT_TEMPLATE in project $PROJECT_CODE_CONTENT_TEMPLATE" const val CGS_SHARE_CONTENT = "share workspace $CONTENT_TEMPLATE in project $PROJECT_CODE_CONTENT_TEMPLATE" const val CGS_EDIT_CONTENT = "edit workspace $CONTENT_TEMPLATE in project $PROJECT_CODE_CONTENT_TEMPLATE" + const val CGS_TOKEN_GENERATE_CONTENT = "generate workspace 1password $CONTENT_TEMPLATE in project $PROJECT_CODE_CONTENT_TEMPLATE" // 云桌面镜像 const val IMAGE_LIST_CONTENT = "list workspace image $CONTENT_TEMPLATE in project $PROJECT_CODE_CONTENT_TEMPLATE" @@ -101,9 +102,10 @@ object ActionAuditContent { const val IMAGE_EDIT_CONTENT = "modify workspace image $CONTENT_TEMPLATE in project $PROJECT_CODE_CONTENT_TEMPLATE" // 代理仓库 - const val CODE_PROXY_CREATE_CONTENT = "create code proxy $CONTENT_TEMPLATE in project $PROJECT_CODE_CONTENT_TEMPLATE" - const val CODE_PROXY_LIST_CONTENT = "list code proxy $CONTENT_TEMPLATE in project $PROJECT_CODE_CONTENT_TEMPLATE" - const val CODE_PROXY_DELETE_CONTENT = "delete code proxy $CONTENT_TEMPLATE in project $PROJECT_CODE_CONTENT_TEMPLATE" + const val TGIT_LINK_CREATE_CONTENT = "create tgit link $CONTENT_TEMPLATE in project $PROJECT_CODE_CONTENT_TEMPLATE" + const val TGIT_LINK_CALLBACK_CREATE_CONTENT = "create tgit link callback $CONTENT_TEMPLATE in project $PROJECT_CODE_CONTENT_TEMPLATE" + const val TGIT_LINK_DELETE_CONTENT = "delete tgit link $CONTENT_TEMPLATE in project $PROJECT_CODE_CONTENT_TEMPLATE" + const val TGIT_LINK_CREATE_PROJECT_CONTENT = "create tgit project $CONTENT_TEMPLATE in project $PROJECT_CODE_CONTENT_TEMPLATE" // 环境 const val ENVIRONMENT_CREATE_CONTENT = "create environment $CONTENT_TEMPLATE in project $PROJECT_CODE_CONTENT_TEMPLATE" diff --git a/src/backend/ci/core/common/common-auth/common-auth-api/src/main/kotlin/com/tencent/devops/common/auth/api/ActionId.kt b/src/backend/ci/core/common/common-auth/common-auth-api/src/main/kotlin/com/tencent/devops/common/auth/api/ActionId.kt index 0b2d3a51653..835cf51999a 100644 --- a/src/backend/ci/core/common/common-auth/common-auth-api/src/main/kotlin/com/tencent/devops/common/auth/api/ActionId.kt +++ b/src/backend/ci/core/common/common-auth/common-auth-api/src/main/kotlin/com/tencent/devops/common/auth/api/ActionId.kt @@ -62,6 +62,7 @@ object ActionId { const val CGS_DELETE = "cgs_delete" const val CGS_SHARE = "cgs_share" const val CGS_EDIT = "cgs_edit" + const val CGS_TOKEN_GENERATE = "cgs_token_generate" // 镜像 const val IMAGE_LIST = "image_list" @@ -69,9 +70,9 @@ object ActionId { const val IMAGE_EDIT = "image_edit" // 代理仓库 - const val CODE_PROXY_CREATE = "code_proxy_create" - const val CODE_PROXY_LIST = "code_proxy_list" - const val CODE_PROXY_DELETE = "code_proxy_delete" + const val TGIT_LINK_CREATE = "tgit_link_create" + const val TGIT_LINK_LIST = "tgit_link_list" + const val TGIT_LINK_DELETE = "tgit_link_delete" // 环境 const val ENVIRONMENT_CREATE = "environment_create" diff --git a/src/backend/ci/core/common/common-auth/common-auth-api/src/main/kotlin/com/tencent/devops/common/auth/api/ResourceTypeId.kt b/src/backend/ci/core/common/common-auth/common-auth-api/src/main/kotlin/com/tencent/devops/common/auth/api/ResourceTypeId.kt index 7dd788d219e..854e9f19caa 100644 --- a/src/backend/ci/core/common/common-auth/common-auth-api/src/main/kotlin/com/tencent/devops/common/auth/api/ResourceTypeId.kt +++ b/src/backend/ci/core/common/common-auth/common-auth-api/src/main/kotlin/com/tencent/devops/common/auth/api/ResourceTypeId.kt @@ -9,7 +9,6 @@ object ResourceTypeId { const val CERT = "cert" const val CGS = "cgs" const val IMAGE = "image" - const val CODE_PROXY = "code_proxy" const val ENVIRONMENT = "environment" const val ENV_NODE = "env_node" const val RULE = "rule" @@ -19,4 +18,5 @@ object ResourceTypeId { const val EXPERIENCE_GROUP = "experience_group" // 自定义 const val SECURITY = "security" + const val TGIT_LINK = "tgit_link" } diff --git a/src/backend/ci/core/common/common-codecc/src/main/kotlin/com/tencent/devops/plugin/codecc/CodeccUtils.kt b/src/backend/ci/core/common/common-codecc/src/main/kotlin/com/tencent/devops/plugin/codecc/CodeccUtils.kt index b840a5962e6..6dc8ab589e5 100644 --- a/src/backend/ci/core/common/common-codecc/src/main/kotlin/com/tencent/devops/plugin/codecc/CodeccUtils.kt +++ b/src/backend/ci/core/common/common-codecc/src/main/kotlin/com/tencent/devops/plugin/codecc/CodeccUtils.kt @@ -31,10 +31,14 @@ object CodeccUtils { const val BK_CI_CODECC_TASK_ID = "BK_CI_CODECC_TASK_ID" + const val BK_CI_CODECC_ATOM_ID_TO_TASK_ID = "BK_CI_CODECC_ATOM_ID_TO_TASK_ID" + const val BK_CI_CODECC_V3_ATOM = "CodeccCheckAtomDebug" const val BK_CI_CODECC_COMMUNITY_ATOM = "CodeCCCheckAtom" + const val BK_CI_CODECC_REPORT_URL = "BK_CI_CODECC_REPORT_URL" + fun isCodeccAtom(atomName: String?): Boolean { return isCodeccNewAtom(atomName) } diff --git a/src/backend/ci/core/common/common-db-base/src/main/kotlin/com/tencent/devops/common/db/utils/JooqUtils.kt b/src/backend/ci/core/common/common-db-base/src/main/kotlin/com/tencent/devops/common/db/utils/JooqUtils.kt index 23c357d61a1..ef23fd7dd89 100644 --- a/src/backend/ci/core/common/common-db-base/src/main/kotlin/com/tencent/devops/common/db/utils/JooqUtils.kt +++ b/src/backend/ci/core/common/common-db-base/src/main/kotlin/com/tencent/devops/common/db/utils/JooqUtils.kt @@ -28,8 +28,6 @@ package com.tencent.devops.common.db.utils import com.mysql.cj.jdbc.exceptions.MySQLTransactionRollbackException -import java.math.BigDecimal -import java.sql.Timestamp import org.jooq.DatePart import org.jooq.Field import org.jooq.Record @@ -37,16 +35,27 @@ import org.jooq.SelectOptionStep import org.jooq.SelectUnionStep import org.jooq.exception.DataAccessException import org.jooq.impl.DSL +import org.springframework.dao.DeadlockLoserDataAccessException +import java.math.BigDecimal +import java.sql.Timestamp object JooqUtils { const val JooqDeadLockMessage = "Deadlock found when trying to get lock; try restarting transaction" - fun retryWhenDeadLock(action: () -> T): T { + fun retryWhenDeadLock(retryTime: Int = 1, action: () -> T): T { return try { action() } catch (dae: DataAccessException) { - if (dae.isDeadLock()) action() else throw dae + if (retryTime - 1 < 0) { + throw dae + } + if (dae.isDeadLock()) retryWhenDeadLock(retryTime - 1, action) else throw dae + } catch (dae: DeadlockLoserDataAccessException) { + if (retryTime - 1 < 0) { + throw dae + } + retryWhenDeadLock(retryTime - 1, action) } } diff --git a/src/backend/ci/core/common/common-db-base/src/test/kotlin/com/tencent/devops/common/db/utils/JooqUtilsTest.kt b/src/backend/ci/core/common/common-db-base/src/test/kotlin/com/tencent/devops/common/db/utils/JooqUtilsTest.kt index 17e2b4b7f82..13612a47f0b 100644 --- a/src/backend/ci/core/common/common-db-base/src/test/kotlin/com/tencent/devops/common/db/utils/JooqUtilsTest.kt +++ b/src/backend/ci/core/common/common-db-base/src/test/kotlin/com/tencent/devops/common/db/utils/JooqUtilsTest.kt @@ -2,6 +2,7 @@ package com.tencent.devops.common.db.utils import org.jooq.exception.DataAccessException import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.DisplayName import org.junit.jupiter.api.Test class JooqUtilsTest { @@ -18,6 +19,19 @@ class JooqUtilsTest { Assertions.assertEquals(expect, actual) } + @Test + @DisplayName("重试多次") + fun retryWhenDeadLock_2() { + var actual = 0 + JooqUtils.retryWhenDeadLock(3) { + if (actual++ < 3) { + throw DataAccessException("mock sql dead lock; ${JooqUtils.JooqDeadLockMessage}") + } + } + val expect = 4 // retry + Assertions.assertEquals(expect, actual) + } + @Test fun assertThrowsRetryWhenDeadLock() { var actual = 0 diff --git a/src/backend/ci/core/common/common-event/src/main/kotlin/com/tencent/devops/common/event/dispatcher/pipeline/mq/MQ.kt b/src/backend/ci/core/common/common-event/src/main/kotlin/com/tencent/devops/common/event/dispatcher/pipeline/mq/MQ.kt index 9077da7266c..e1c8a7d0a8f 100644 --- a/src/backend/ci/core/common/common-event/src/main/kotlin/com/tencent/devops/common/event/dispatcher/pipeline/mq/MQ.kt +++ b/src/backend/ci/core/common/common-event/src/main/kotlin/com/tencent/devops/common/event/dispatcher/pipeline/mq/MQ.kt @@ -118,6 +118,7 @@ object MQ { const val QUEUE_PIPELINE_BUILD_FINISH_GITCI = "q.engine.pipeline.build.gitci" const val QUEUE_PIPELINE_BUILD_FINISH_LOG = "q.engine.pipeline.build.log" const val QUEUE_PIPELINE_BUILD_FINISH_SUBPIPEINE = "q.engine.pipeline.build.subpipeline" + const val QUEUE_PIPELINE_BUILD_QUEUE_SUBPIPEINE = "q.engine.pipeline.build.queue.subpipeline" const val QUEUE_PIPELINE_BUILD_FINISH_WEBHOOK_QUEUE = "q.engine.pipeline.build.finish.webhook.queue" const val QUEUE_PIPELINE_BUILD_FINISH_NOTIFY_QUEUE = "q.engine.pipeline.build.finish.notify.queue" @@ -270,6 +271,7 @@ object MQ { // 回调 const val EXCHANGE_PIPELINE_BUILD_CALL_BACK_FANOUT = "e.engine.pipeline.build.callback.fanout" const val QUEUE_PIPELINE_BUILD_STATUS_CHANGE = "e.engine.pipeline.build.callback.change" + const val QUEUE_PIPELINE_BUILD_STATUS_METRICS = "q.engine.pipeline.build.callback.metrics" // 蓝盾项目管理 const val EXCHANGE_PROJECT_CREATE_FANOUT = "e.project.create.exchange.fanout" diff --git a/src/backend/ci/core/common/common-event/src/main/kotlin/com/tencent/devops/common/event/pojo/pipeline/PipelineBuildStatusBroadCastEvent.kt b/src/backend/ci/core/common/common-event/src/main/kotlin/com/tencent/devops/common/event/pojo/pipeline/PipelineBuildStatusBroadCastEvent.kt index 7be78203f8d..dc9c9cbfd2a 100644 --- a/src/backend/ci/core/common/common-event/src/main/kotlin/com/tencent/devops/common/event/pojo/pipeline/PipelineBuildStatusBroadCastEvent.kt +++ b/src/backend/ci/core/common/common-event/src/main/kotlin/com/tencent/devops/common/event/pojo/pipeline/PipelineBuildStatusBroadCastEvent.kt @@ -30,6 +30,7 @@ package com.tencent.devops.common.event.pojo.pipeline import com.tencent.devops.common.event.annotation.Event import com.tencent.devops.common.event.dispatcher.pipeline.mq.MQ import com.tencent.devops.common.event.enums.ActionType +import java.time.LocalDateTime /** * 构建状态的广播事件，用于通知等 @@ -42,7 +43,14 @@ data class PipelineBuildStatusBroadCastEvent( override val userId: String, val buildId: String, val stageId: String? = null, + val containerHashId: String? = null, + val jobId: String? = null, val taskId: String? = null, + val stepId: String? = null, + val executeCount: Int?, + val buildStatus: String?, + val atomCode: String? = null, + val eventTime: LocalDateTime? = LocalDateTime.now(), override var actionType: ActionType, override var delayMills: Int = 0 ) : IPipelineEvent(actionType, source, projectId, pipelineId, userId, delayMills) diff --git a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/ContextTree.kt b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/ContextTree.kt new file mode 100644 index 00000000000..204275122e6 --- /dev/null +++ b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/ContextTree.kt @@ -0,0 +1,253 @@ +package com.tencent.devops.common.expression + +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.ArrayNode +import com.fasterxml.jackson.databind.node.JsonNodeFactory +import com.fasterxml.jackson.databind.node.ObjectNode +import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import com.tencent.devops.common.expression.context.ContextValueNode +import com.tencent.devops.common.expression.context.DictionaryContextData +import com.tencent.devops.common.expression.context.DictionaryContextDataWithVal +import com.tencent.devops.common.expression.context.PipelineContextData +import com.tencent.devops.common.expression.context.StringContextData +import com.tencent.devops.common.expression.expression.sdk.NamedValueInfo +import java.util.LinkedList +import java.util.Queue +import java.util.Stack +import kotlin.jvm.Throws + +/** + * 用来将流水线变量转为树的形式，来对其转换到表达式引擎做兼容处理 + * 如 a.b.c = 1 的变量转换后就是 a->b->c = 1 + * 树中节点转换时默认会使用 map，除非其节点对应的json明确声明了当前节点是数组 + * @param nodes 所有的树根节点 + */ +@Suppress("ComplexMethod") +class ContextTree( + val nodes: MutableMap = mutableMapOf() +) { + fun addNode(key: String, value: String) { + val tokens = key.split(".") + val rootKey = tokens.first() + // 如果之前的节点中没有根节点直接添加 + val rootNode = nodes[rootKey] + if (rootNode == null) { + nodes[rootKey] = toTree(tokens, value) + return + } else if (tokens.size == 1) { + rootNode.value = value + return + } + // 根节点已经存在了需要加入进去，同时做校验 + // 1、命中存在 key 的节点，无值赋值，有值报错 + // 2、命中没有 key 的节点，在最后一个找不到 key 的父结点下添加没找到的 key 的树 + var tokenIdx = 1 + var parentNode: ContextTreeNode = rootNode + var valueNode: ContextTreeNode? = null + while (tokenIdx < tokens.size) { + var endFlag = true + parentNode.breadthFirstTraversal { n -> + // 还没有找到当前层 + if (n.depth < tokenIdx) { + return@breadthFirstTraversal false + } + // 过了当前层还没找到说明需要在这一层的父结点上面加从这一层开始的内容 + if (n.depth > tokenIdx) { + return@breadthFirstTraversal true + } + // 在当前层找,如果能找到就继续找这个节点的下一层 + // 如果是token最后一层则赋值，如果存在原本有值的情况则报错 + if (n.key == tokens[tokenIdx]) { + if (tokenIdx != tokens.lastIndex) { + tokenIdx++ + parentNode = n + endFlag = false + return@breadthFirstTraversal true + } + if (n.value != null && n.value != value) { + throw ContextDataRuntimeException("duplicate key ${n.key} value ${n.value}|$value not equal") + } + valueNode = n + return@breadthFirstTraversal true + } + parentNode = n.parent ?: rootNode + return@breadthFirstTraversal false + } + if (endFlag) { + break + } + } + + if (valueNode != null) { + valueNode!!.value = value + } else { + parentNode.addChild(toTree(tokens.subList(tokenIdx, tokens.size), value, parentNode.depth + 1)) + } + } + + /** + * 将树中节点转换为上下文并添加到输入参数 + * @param context 被添加的上下文 + * @param nameValue 被添加的命名 + */ + fun toContext(context: DictionaryContextData, nameValue: MutableList) { + nodes.forEach { (key, value) -> + context[key] = value.toContext() + nameValue.add(NamedValueInfo(key, ContextValueNode())) + } + } + + private fun toTree(tokens: List, value: String, depth: Int = 0): ContextTreeNode { + if (tokens.size == 1) { + return ContextTreeNode(tokens[0], value) + } + val rNode = ContextTreeNode(tokens[0], null, depth = depth) + val valueTokens = tokens.subList(1, tokens.size) + var node = rNode + valueTokens.forEachIndexed { idx, t -> + val child = ContextTreeNode( + key = t, + value = if (idx == valueTokens.lastIndex) { + value + } else { + null + } + ) + node.addChild(child) + node = child + } + return rNode + } +} + +/** + * 树节点 + * @param key 变量名 + * @param value 变量值，存在为空情况即用户没有具体存入值但是拥有计算逻辑的节点，如 a.b.c = 1 中的a和b + * @param array 当前节点是否是数组节点 + * @param parent 父节点 + * @param children 所有子节点 + * @param depth 深度 + */ +open class ContextTreeNode( + val key: String, + var value: String?, + private val array: Boolean = false, + var parent: ContextTreeNode? = null, + private val children: MutableList = mutableListOf(), + var depth: Int = 0 +) { + + fun addChild(child: ContextTreeNode) { + child.parent = this + child.depth = depth + 1 + children.add(child) + } + + fun depthFirstTraversal(run: (node: ContextTreeNode) -> Unit) { + val stack = Stack() + stack.push(this) + + while (!stack.isEmpty()) { + val node = stack.pop() + + run(node) + + // 将子节点逆序入栈，保证先访问左边的子节点 + for (i in node.children.lastIndex downTo 0) { + stack.push(node.children[i]) + } + } + } + + fun breadthFirstTraversal(run: (node: ContextTreeNode) -> Boolean) { + val queue: Queue = LinkedList() + queue.offer(this) + + while (!queue.isEmpty()) { + val node = queue.poll() + + if (run(node)) { + return + } + + for (child in node.children) { + queue.offer(child) + } + } + } + + @Throws(ContextJsonFormatException::class) + fun toContext(): PipelineContextData { + if (this.children.isEmpty()) { + return StringContextData(this.value ?: "") + } + val dict = value?.let { DictionaryContextDataWithVal(it) } ?: DictionaryContextData() + this.children.forEach { child -> + dict[child.key] = child.toContext() + } + return dict + } + + // 校验当前节点的值转换的JSON树是否与子节点结构和值相同 +// @Throws(ContextJsonFormatException::class) +// private fun checkJson() { +// val jsonTree = try { +// ObjectMapper().readTree(this.value) +// } catch (e: Exception) { +// throw ContextJsonFormatException("${this.value} to json error ${e.localizedMessage}") +// } +// // TODO: 是否需要兼容 json 不同类型 +// jsonTree.equals(ObjectNodeComparator(), this.toJson()) +// } + + private fun toJson(): JsonNode { + val jsonNodeFactory = JsonNodeFactory.instance + val rootObj = jacksonObjectMapper().createObjectNode() + if (this.children.isEmpty()) { + return jsonNodeFactory.textNode(this.value) + } + this.children.forEach { child -> + rootObj.putIfAbsent(child.key, child.toJson()) + } + return rootObj + } +} + +// 存在用户的 json 值为 array 但是翻译成了 map，这里我们认为是等价的 +class ObjectNodeComparator : Comparator { + override fun compare(o1: JsonNode?, o2: JsonNode?): Int { + if (o1 == null && o2 == null) { + return 0 + } + if (o1 == null || o2 == null) { + return 1 + } + if (o1 == o2) { + return 0 + } + if (o1 is ArrayNode && o2 is ObjectNode) { + if (o1.size() != o2.size()) { + return 1 + } + o1.forEachIndexed { index, node -> + if (o2[index] == null || node != o2[index]) { + return 1 + } + } + return 0 + } + if (o1 is ObjectNode && o2 is ArrayNode) { + if (o1.size() != o2.size()) { + return 1 + } + o2.forEachIndexed { index, node -> + if (o1[index] == null || node != o1[index]) { + return 1 + } + } + return 0 + } + return 1 + } +} diff --git a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/ExpressionException.kt b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/ExpressionException.kt index 8a53b94471d..37d42ccbd2c 100644 --- a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/ExpressionException.kt +++ b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/ExpressionException.kt @@ -89,3 +89,5 @@ class ContextNotFoundException(override val message: String?) : ExpressionExcept ) } } + +class ContextJsonFormatException(override val message: String?) : ExpressionException() diff --git a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/ExpressionParser.kt b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/ExpressionParser.kt index 457eb64c522..7e473d70363 100644 --- a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/ExpressionParser.kt +++ b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/ExpressionParser.kt @@ -30,7 +30,6 @@ package com.tencent.devops.common.expression import com.tencent.devops.common.expression.context.ContextValueNode import com.tencent.devops.common.expression.context.DictionaryContextData import com.tencent.devops.common.expression.context.PipelineContextData -import com.tencent.devops.common.expression.context.StringContextData import com.tencent.devops.common.expression.expression.ExpressionConstants import com.tencent.devops.common.expression.expression.IExpressionNode import com.tencent.devops.common.expression.expression.IFunctionInfo @@ -102,44 +101,22 @@ object ExpressionParser { return if (result.value is PipelineContextData) result.value.fetchValue() else result.value } + /** + * 将流水线变量转换为表达式上下文类型，存在如下情况 + * 1、a = str, 直接使用 string 类型的上下文保存即可 + * 2、a.b.c = str, 将 a.b.c 升格为上下文中的嵌套 map 保存 既 a {b: {c: str}} + * 3、a.b.c = str 且 a.b = {"c": "str"}, 需要校验 a.b 所保存的 json 与 a.b.c 结构和数据是否相同后再升格 + */ fun fillContextByMap( contextMap: Map, context: ExecutionContext, nameValue: MutableList ) { - contextMap.forEach { (key, value) -> - var data: DictionaryContextData? = null - val tokens = key.split('.') - if (tokens.size > 1) { - tokens.forEachIndexed { index, token -> - if (index == tokens.size - 1) { - data!!.add(token, StringContextData(value)) - return@forEachIndexed - } - - if (index == 0) { - if (context.expressionValues[token] != null) { - data = context.expressionValues[token] as DictionaryContextData - return@forEachIndexed - } - nameValue.add(NamedValueInfo(token, ContextValueNode())) - context.expressionValues[token] = DictionaryContextData() - data = context.expressionValues[token] as DictionaryContextData - return@forEachIndexed - } - - data!![token]?.let { - data = it as DictionaryContextData - return@forEachIndexed - } - data!![token] = DictionaryContextData() - data = data!![token] as DictionaryContextData - } - } else { - nameValue.add(NamedValueInfo(key, ContextValueNode())) - context.expressionValues[key] = StringContextData(value) - } + val contextTree = ContextTree() + contextMap.forEach { (k, v) -> + contextTree.addNode(k, v) } + contextTree.toContext(context.expressionValues, nameValue) } fun createTree( diff --git a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/AbsDictionaryContextData.kt b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/AbsDictionaryContextData.kt new file mode 100644 index 00000000000..0de7a0d8bf7 --- /dev/null +++ b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/AbsDictionaryContextData.kt @@ -0,0 +1,115 @@ +package com.tencent.devops.common.expression.context + +import com.fasterxml.jackson.databind.JsonNode +import com.tencent.devops.common.expression.expression.sdk.IReadOnlyObject +import com.tencent.devops.common.expression.utils.ExpressionJsonUtil +import java.util.TreeMap + +/** + * dict 的抽象类，总结公共方法 + */ +abstract class AbsDictionaryContextData : + PipelineContextData(PipelineContextDataType.DICTIONARY), + Iterable>, + IReadOnlyObject { + + protected open var mIndexLookup: TreeMap? = null + protected open var mList: MutableList = mutableListOf() + + override val values: Iterable + get() { + if (mList.isNotEmpty()) { + return mList.map { it.value } + } + return emptyList() + } + + override fun tryGetValue(key: String): Pair { + if (mList.isNotEmpty() && indexLookup.containsKey(key)) { + return Pair(mList[indexLookup[key]!!].value, true) + } + + return Pair(null, false) + } + + protected val indexLookup: MutableMap + get() { + if (mIndexLookup == null) { + mIndexLookup = TreeMap() + if (mList.isNotEmpty()) { + mList.forEachIndexed { index, pair -> + mIndexLookup!![pair.key] = index + } + } + } + + return mIndexLookup!! + } + + private val list: MutableList + get() { + return mList + } + + operator fun set(k: String, value: PipelineContextData?) { + // Existing + val index = indexLookup[k] + if (index != null) { + val key = mList[index].key // preserve casing + mList[index] = DictionaryContextDataPair(key, value) + } + // New + else { + add(k, value) + } + } + + open operator fun get(k: String): PipelineContextData? { + // Existing + val index = indexLookup[k] ?: return null + return list[index].value + } + + open operator fun IReadOnlyObject.get(key: String): Any? { + val index = indexLookup[key] ?: return null + return list[index].value + } + + operator fun Pair.get(key: Int): Pair { + val pair = mList[key] + return Pair(pair.key, pair.value) + } + + fun add(pairs: Iterable>) { + pairs.forEach { pair -> + add(pair.first, pair.second) + } + } + + fun add( + key: String, + value: PipelineContextData? + ) { + indexLookup[key] = mList.count() + list.add(DictionaryContextDataPair(key, value)) + } + + override fun iterator(): Iterator> { + return mList.map { pair -> Pair(pair.key, pair.value); }.iterator() + } + + override fun toJson(): JsonNode { + val json = ExpressionJsonUtil.createObjectNode() + if (mList.isNotEmpty()) { + mList.forEach { + json.set(it.key, it.value?.toJson()) + } + } + return json + } + + protected data class DictionaryContextDataPair( + val key: String, + val value: PipelineContextData? + ) +} diff --git a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/DictionaryContextData.kt b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/DictionaryContextData.kt index f82353393f9..ff13dfc165c 100644 --- a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/DictionaryContextData.kt +++ b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/DictionaryContextData.kt @@ -27,96 +27,11 @@ package com.tencent.devops.common.expression.context -import com.fasterxml.jackson.databind.JsonNode -import com.tencent.devops.common.expression.expression.sdk.IReadOnlyObject -import com.tencent.devops.common.expression.utils.ExpressionJsonUtil import java.util.TreeMap -open class DictionaryContextData : - PipelineContextData(PipelineContextDataType.DICTIONARY), - Iterable>, - IReadOnlyObject { - - protected open var mIndexLookup: TreeMap? = null - protected open var mList: MutableList = mutableListOf() - - override val values: Iterable - get() { - if (mList.isNotEmpty()) { - return mList.map { it.value } - } - return emptyList() - } - - override fun tryGetValue(key: String): Pair { - if (mList.isNotEmpty() && indexLookup.containsKey(key)) { - return Pair(mList[indexLookup[key]!!].value, true) - } - - return Pair(null, false) - } - - protected val indexLookup: MutableMap - get() { - if (mIndexLookup == null) { - mIndexLookup = TreeMap() - if (mList.isNotEmpty()) { - mList.forEachIndexed { index, pair -> - mIndexLookup!![pair.key] = index - } - } - } - - return mIndexLookup!! - } - - private val list: MutableList - get() { - return mList - } - - operator fun set(k: String, value: PipelineContextData?) { - // Existing - val index = indexLookup[k] - if (index != null) { - val key = mList[index].key // preserve casing - mList[index] = DictionaryContextDataPair(key, value) - } - // New - else { - add(k, value) - } - } - - open operator fun get(k: String): PipelineContextData? { - // Existing - val index = indexLookup[k] ?: return null - return list[index].value - } - - open operator fun IReadOnlyObject.get(key: String): Any? { - val index = indexLookup[key] ?: return null - return list[index].value - } - - operator fun Pair.get(key: Int): Pair { - val pair = mList[key] - return Pair(pair.key, pair.value) - } - - fun add(pairs: Iterable>) { - pairs.forEach { pair -> - add(pair.first, pair.second) - } - } - - fun add( - key: String, - value: PipelineContextData? - ) { - indexLookup[key] = mList.count() - list.add(DictionaryContextDataPair(key, value)) - } +open class DictionaryContextData : AbsDictionaryContextData() { + override var mIndexLookup: TreeMap? = null + override var mList: MutableList = mutableListOf() override fun clone(): PipelineContextData { val result = DictionaryContextData() @@ -131,16 +46,6 @@ open class DictionaryContextData : return result } - override fun toJson(): JsonNode { - val json = ExpressionJsonUtil.createObjectNode() - if (mList.isNotEmpty()) { - mList.forEach { - json.set(it.key, it.value?.toJson()) - } - } - return json - } - override fun fetchValue(): Map { val map = mutableMapOf() if (mList.isNotEmpty()) { @@ -150,13 +55,4 @@ open class DictionaryContextData : } return map } - - override fun iterator(): Iterator> { - return mList.map { pair -> Pair(pair.key, pair.value); }.iterator() - } - - protected data class DictionaryContextDataPair( - val key: String, - val value: PipelineContextData? - ) -} +} \ No newline at end of file diff --git a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/DictionaryContextDataWithVal.kt b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/DictionaryContextDataWithVal.kt new file mode 100644 index 00000000000..15bc0e3ba58 --- /dev/null +++ b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/context/DictionaryContextDataWithVal.kt @@ -0,0 +1,36 @@ +package com.tencent.devops.common.expression.context + +import com.tencent.devops.common.expression.utils.ExpressionJsonUtil +import java.util.TreeMap + +/** + * 包含用户原始值的类型，用来兼容老数据中 a.b.c 已经存在一个固定值的情况，优先兼容老的 json 存储 + * @param oriValue 用户原始值 + */ +class DictionaryContextDataWithVal( + private val oriValue: String +) : AbsDictionaryContextData() { + override var mIndexLookup: TreeMap? = null + override var mList: MutableList = mutableListOf() + + override fun clone(): PipelineContextData { + val result = DictionaryContextDataWithVal(oriValue) + + if (mList.isNotEmpty()) { + result.mList = mutableListOf() + mList.forEach { + result.mList.add(DictionaryContextDataPair(it.key, it.value?.clone())) + } + } + + return result + } + + override fun fetchValue(): Any { + return try { + ExpressionJsonUtil.getObjectMapper().readTree(oriValue) + } catch (ignore: Exception) { + return oriValue + } + } +} \ No newline at end of file diff --git a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/ExpressionConstants.kt b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/ExpressionConstants.kt index 04a6c1931a5..c879084b5c9 100644 --- a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/ExpressionConstants.kt +++ b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/ExpressionConstants.kt @@ -34,6 +34,7 @@ import com.tencent.devops.common.expression.expression.functions.FromJson import com.tencent.devops.common.expression.expression.functions.Join import com.tencent.devops.common.expression.expression.functions.StartsWith import com.tencent.devops.common.expression.expression.functions.StrToTime +import com.tencent.devops.common.expression.expression.functions.ToJson import com.tencent.devops.common.expression.expression.sdk.Function import java.util.TreeMap @@ -46,6 +47,7 @@ object ExpressionConstants { addFunction(EndsWith.name, 2, 2, EndsWith()) addFunction(StartsWith.name, 2, 2, StartsWith()) addFunction(FromJson.name, 1, 1, FromJson()) + addFunction(ToJson.name, 1, 1, ToJson()) addFunction(StrToTime.name, 1, 1, StrToTime()) addFunction(Join.name, 1, 2, Join()) addFunction(Format.name, 1, Byte.MAX_VALUE.toInt(), Format()) diff --git a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/functions/ToJson.kt b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/functions/ToJson.kt new file mode 100644 index 00000000000..ab891a5c22a --- /dev/null +++ b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/functions/ToJson.kt @@ -0,0 +1,35 @@ +package com.tencent.devops.common.expression.expression.functions + +import com.tencent.devops.common.api.util.JsonUtil +import com.tencent.devops.common.expression.context.PipelineContextData +import com.tencent.devops.common.expression.context.StringContextData +import com.tencent.devops.common.expression.expression.sdk.EvaluationContext +import com.tencent.devops.common.expression.expression.sdk.Function +import com.tencent.devops.common.expression.expression.sdk.ResultMemory +import com.tencent.devops.common.expression.utils.ExpressionJsonUtil + +class ToJson : Function() { + companion object { + const val name = "toJSON" + } + + override fun createNode(): Function = ToJson() + + override fun evaluateCore(context: EvaluationContext): Pair { + val res = parameters[0].evaluate(context) + if (res.value == null) { + return null to null + } + val json = if (res.value is PipelineContextData) { + ExpressionJsonUtil.write(res.value.toJson()) + } else { + JsonUtil.toJson(res.value) + } + return Pair(null, StringContextData(json)) + } + + override fun subNameValueEvaluateCore(context: EvaluationContext): Pair { + val left = parameters[0].subNameValueEvaluate(context).parseSubNameValueEvaluateResult() + return Pair("$name($left)", false) + } +} \ No newline at end of file diff --git a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/tokens/LexicalAnalyzer.kt b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/tokens/LexicalAnalyzer.kt index 5fb4bc30e6e..1df81724eee 100644 --- a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/tokens/LexicalAnalyzer.kt +++ b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/expression/tokens/LexicalAnalyzer.kt @@ -40,7 +40,10 @@ class LexicalAnalyzer(private val expression: String, val subNameValueEvaluateIn // 当前遍历到的字符 private var mIndex: Int = 0 - // 上一个便利的 token + // 上上一个遍历的 token + private var mLastLastToken: Token? = null + + // 上一个遍历的 token private var mLastToken: Token? = null fun tryGetNextToken(): Pair { @@ -114,12 +117,12 @@ class LexicalAnalyzer(private val expression: String, val subNameValueEvaluateIn mLastToken?.kind == TokenKind.StartParameters || // "(" function call mLastToken?.kind == TokenKind.LogicalOperator ) // "!", "==", etc - { - rToken = readNumberToken() - } + { + rToken = readNumberToken() + } // "." else { - rToken = createToken(TokenKind.Dereference, c, mIndex++) + rToken = readDereferenceToken() } } else if (c == '-' || c == '+' || (c in '0'..'9')) { rToken = readNumberToken() @@ -132,23 +135,52 @@ class LexicalAnalyzer(private val expression: String, val subNameValueEvaluateIn } } + mLastLastToken = mLastToken mLastToken = rToken return Pair(rToken, true) } private fun readNumberToken(): Token { val startIndex = mIndex - do { + var numberIndex = false + while (true) { mIndex++ - } while (mIndex < expression.length && (!testTokenBoundary(expression[mIndex]) || expression[mIndex] == '.')) + if (mIndex >= expression.length) { + // 兼容 xxx.number + if (mLastToken?.kind == TokenKind.Dereference) { + numberIndex = true + } + break + } + if (testTokenBoundary(expression[mIndex]) && expression[mIndex] != '.') { + // 兼容 xxx.number + if (mLastToken?.kind == TokenKind.Dereference) { + numberIndex = true + } + break + } + // 兼容下 xxx.number.xxx + if (mLastToken?.kind == TokenKind.Dereference && expression[mIndex] == '.') { + numberIndex = true + break + } + } val length = mIndex - startIndex val str = expression.substring(startIndex, startIndex + length) val d = ExpressionUtility.parseNumber(str) return if (d.isNaN()) { createToken(TokenKind.Unexpected, str, startIndex) } else { - createToken(TokenKind.Number, str, startIndex, d) + createToken(TokenKind.Number, str, startIndex, d, numberIndex) + } + } + + private fun readDereferenceToken(): Token { + // 兼容下 xxx.number.xxx 的情况 + if (mLastToken?.kind == TokenKind.Number && mLastLastToken?.kind == TokenKind.Dereference) { + return createToken(TokenKind.Dereference, expression[mIndex], mIndex++, null, true) } + return createToken(TokenKind.Dereference, expression[mIndex], mIndex++) } private fun readKeywordToken(): Token { @@ -189,9 +221,9 @@ class LexicalAnalyzer(private val expression: String, val subNameValueEvaluateIn // Function if (tempIndex < expression.length && expression[tempIndex] == ExpressionConstants.START_GROUP) // "(" - { - createToken(TokenKind.Function, str, startIndex) - } else { + { + createToken(TokenKind.Function, str, startIndex) + } else { createToken(TokenKind.NamedValue, str, startIndex) } } else { @@ -314,211 +346,218 @@ class LexicalAnalyzer(private val expression: String, val subNameValueEvaluateIn kind: TokenKind, rawValue: Char, index: Int, - parsedValue: Any? = null + parsedValue: Any? = null, + // 很确定当前生成的 token 是合法跳过校验 + skipCheck: Boolean = false ): Token { - return createToken(kind, rawValue.toString(), index, parsedValue) + return createToken(kind, rawValue.toString(), index, parsedValue, skipCheck) } private fun createToken( kind: TokenKind, rawValue: String, index: Int, - parsedValue: Any? = null + parsedValue: Any? = null, + // 很确定当前生成的 token 是合法跳过校验 + skipCheck: Boolean = false ): Token { // 根据上一个token判断当前token是否合法 var legal = false - when (kind) { - // "(" logical grouping - TokenKind.StartGroup -> { - // 是第一个还是在“，”或“（”或“[”或逻辑运算符之后 - legal = checkLastToken( - null, - TokenKind.Separator, - TokenKind.StartGroup, - TokenKind.StartParameters, - TokenKind.StartIndex, - TokenKind.LogicalOperator - ) - } - // "[" - TokenKind.StartIndex -> { - // Follows ")", "]", "*", a property name, or a named-value - legal = checkLastToken( - TokenKind.EndGroup, - TokenKind.EndParameters, - TokenKind.EndIndex, - TokenKind.Wildcard, - TokenKind.PropertyName, - TokenKind.NamedValue - ) - } - // "(" function call - TokenKind.StartParameters -> { - // Follows a function - legal = checkLastToken(TokenKind.Function) - } - // ")" logical grouping - TokenKind.EndGroup -> { - // Follows ")", "]", "*", a literal, a property name, or a named-value - legal = checkLastToken( - TokenKind.EndGroup, - TokenKind.EndParameters, - TokenKind.EndIndex, - TokenKind.Wildcard, - TokenKind.Null, - TokenKind.Boolean, - TokenKind.Number, - TokenKind.String, - TokenKind.PropertyName, - TokenKind.NamedValue - ) - } - // "]" - TokenKind.EndIndex -> { - // Follows ")", "]", "*", a literal, a property name, or a named-value - legal = checkLastToken( - TokenKind.EndGroup, - TokenKind.EndParameters, - TokenKind.EndIndex, - TokenKind.Wildcard, - TokenKind.Null, - TokenKind.Boolean, - TokenKind.Number, - TokenKind.String, - TokenKind.PropertyName, - TokenKind.NamedValue - ) - } - // ")" function call - TokenKind.EndParameters -> { - // Follows "(" function call, ")", "]", "*", a literal, a property name, or a named-value - legal = checkLastToken( - TokenKind.StartParameters, - TokenKind.EndGroup, - TokenKind.EndParameters, - TokenKind.EndIndex, - TokenKind.Wildcard, - TokenKind.Null, - TokenKind.Boolean, - TokenKind.Number, - TokenKind.String, - TokenKind.PropertyName, - TokenKind.NamedValue - ) - } - // "," - TokenKind.Separator -> { - // Follows ")", "]", "*", a literal, a property name, or a named-value - legal = checkLastToken( - TokenKind.EndGroup, - TokenKind.EndParameters, - TokenKind.EndIndex, - TokenKind.Wildcard, - TokenKind.Null, - TokenKind.Boolean, - TokenKind.Number, - TokenKind.String, - TokenKind.PropertyName, - TokenKind.NamedValue - ) - } - // "." - TokenKind.Dereference -> { - // Follows ")", "]", "*", a property name, or a named-value - legal = checkLastToken( - TokenKind.EndGroup, - TokenKind.EndParameters, - TokenKind.EndIndex, - TokenKind.Wildcard, - TokenKind.PropertyName, - TokenKind.NamedValue - ) - } - // "*" - TokenKind.Wildcard -> { - // Follows "[" or "." - legal = checkLastToken(TokenKind.StartIndex, TokenKind.Dereference) - } - // "!", "==", etc - TokenKind.LogicalOperator -> { - when (rawValue) { - ExpressionConstants.NOT -> { - // Is first or follows "," or "(" or "[" or a logical operator - legal = checkLastToken( - null, - TokenKind.Separator, - TokenKind.StartGroup, - TokenKind.StartParameters, - TokenKind.StartIndex, - TokenKind.LogicalOperator - ) - } + if (skipCheck) { + legal = true + } else { + when (kind) { + // "(" logical grouping + TokenKind.StartGroup -> { + // 是第一个还是在“，”或“（”或“[”或逻辑运算符之后 + legal = checkLastToken( + null, + TokenKind.Separator, + TokenKind.StartGroup, + TokenKind.StartParameters, + TokenKind.StartIndex, + TokenKind.LogicalOperator + ) + } + // "[" + TokenKind.StartIndex -> { + // Follows ")", "]", "*", a property name, or a named-value + legal = checkLastToken( + TokenKind.EndGroup, + TokenKind.EndParameters, + TokenKind.EndIndex, + TokenKind.Wildcard, + TokenKind.PropertyName, + TokenKind.NamedValue + ) + } + // "(" function call + TokenKind.StartParameters -> { + // Follows a function + legal = checkLastToken(TokenKind.Function) + } + // ")" logical grouping + TokenKind.EndGroup -> { + // Follows ")", "]", "*", a literal, a property name, or a named-value + legal = checkLastToken( + TokenKind.EndGroup, + TokenKind.EndParameters, + TokenKind.EndIndex, + TokenKind.Wildcard, + TokenKind.Null, + TokenKind.Boolean, + TokenKind.Number, + TokenKind.String, + TokenKind.PropertyName, + TokenKind.NamedValue + ) + } + // "]" + TokenKind.EndIndex -> { + // Follows ")", "]", "*", a literal, a property name, or a named-value + legal = checkLastToken( + TokenKind.EndGroup, + TokenKind.EndParameters, + TokenKind.EndIndex, + TokenKind.Wildcard, + TokenKind.Null, + TokenKind.Boolean, + TokenKind.Number, + TokenKind.String, + TokenKind.PropertyName, + TokenKind.NamedValue + ) + } + // ")" function call + TokenKind.EndParameters -> { + // Follows "(" function call, ")", "]", "*", a literal, a property name, or a named-value + legal = checkLastToken( + TokenKind.StartParameters, + TokenKind.EndGroup, + TokenKind.EndParameters, + TokenKind.EndIndex, + TokenKind.Wildcard, + TokenKind.Null, + TokenKind.Boolean, + TokenKind.Number, + TokenKind.String, + TokenKind.PropertyName, + TokenKind.NamedValue + ) + } + // "," + TokenKind.Separator -> { + // Follows ")", "]", "*", a literal, a property name, or a named-value + legal = checkLastToken( + TokenKind.EndGroup, + TokenKind.EndParameters, + TokenKind.EndIndex, + TokenKind.Wildcard, + TokenKind.Null, + TokenKind.Boolean, + TokenKind.Number, + TokenKind.String, + TokenKind.PropertyName, + TokenKind.NamedValue + ) + } + // "." + TokenKind.Dereference -> { + // Follows ")", "]", "*", a property name, or a named-value + legal = checkLastToken( + TokenKind.EndGroup, + TokenKind.EndParameters, + TokenKind.EndIndex, + TokenKind.Wildcard, + TokenKind.PropertyName, + TokenKind.NamedValue + ) + } + // "*" + TokenKind.Wildcard -> { + // Follows "[" or "." + legal = checkLastToken(TokenKind.StartIndex, TokenKind.Dereference) + } + // "!", "==", etc + TokenKind.LogicalOperator -> { + when (rawValue) { + ExpressionConstants.NOT -> { + // Is first or follows "," or "(" or "[" or a logical operator + legal = checkLastToken( + null, + TokenKind.Separator, + TokenKind.StartGroup, + TokenKind.StartParameters, + TokenKind.StartIndex, + TokenKind.LogicalOperator + ) + } - else -> { - // Follows ")", "]", "*", a literal, a property name, or a named-value - legal = checkLastToken( - TokenKind.EndGroup, - TokenKind.EndParameters, - TokenKind.EndIndex, - TokenKind.Wildcard, - TokenKind.Null, - TokenKind.Boolean, - TokenKind.Number, - TokenKind.String, - TokenKind.PropertyName, - TokenKind.NamedValue - ) + else -> { + // Follows ")", "]", "*", a literal, a property name, or a named-value + legal = checkLastToken( + TokenKind.EndGroup, + TokenKind.EndParameters, + TokenKind.EndIndex, + TokenKind.Wildcard, + TokenKind.Null, + TokenKind.Boolean, + TokenKind.Number, + TokenKind.String, + TokenKind.PropertyName, + TokenKind.NamedValue + ) + } } } - } - TokenKind.Null, TokenKind.Boolean, TokenKind.Number, TokenKind.String -> { - // Is first or follows "," or "[" or "(" or a logical operator (e.g. "!" or "==" etc) - legal = checkLastToken( - null, - TokenKind.Separator, - TokenKind.StartIndex, - TokenKind.StartGroup, - TokenKind.StartParameters, - TokenKind.LogicalOperator - ) - } + TokenKind.Null, TokenKind.Boolean, TokenKind.Number, TokenKind.String -> { + // Is first or follows "," or "[" or "(" or a logical operator (e.g. "!" or "==" etc) + legal = checkLastToken( + null, + TokenKind.Separator, + TokenKind.StartIndex, + TokenKind.StartGroup, + TokenKind.StartParameters, + TokenKind.LogicalOperator + ) + } - // 针对subNameValue特供，在expression后都是合法的 - TokenKind.Expression -> { - legal = true - } + // 针对subNameValue特供，在expression后都是合法的 + TokenKind.Expression -> { + legal = true + } - TokenKind.PropertyName -> { - // Follows "." - legal = checkLastToken(TokenKind.Dereference) - } + TokenKind.PropertyName -> { + // Follows "." + legal = checkLastToken(TokenKind.Dereference) + } - TokenKind.Function -> { - // Is first or follows "," or "[" or "(" or a logical operator (e.g. "!" or "==" etc) - legal = checkLastToken( - null, - TokenKind.Separator, - TokenKind.StartIndex, - TokenKind.StartGroup, - TokenKind.StartParameters, - TokenKind.LogicalOperator - ) - } + TokenKind.Function -> { + // Is first or follows "," or "[" or "(" or a logical operator (e.g. "!" or "==" etc) + legal = checkLastToken( + null, + TokenKind.Separator, + TokenKind.StartIndex, + TokenKind.StartGroup, + TokenKind.StartParameters, + TokenKind.LogicalOperator + ) + } - TokenKind.NamedValue -> { - // Is first or follows "," or "[" or "(" or a logical operator (e.g. "!" or "==" etc) - legal = checkLastToken( - null, - TokenKind.Separator, - TokenKind.StartIndex, - TokenKind.StartGroup, - TokenKind.StartParameters, - TokenKind.LogicalOperator - ) + TokenKind.NamedValue -> { + // Is first or follows "," or "[" or "(" or a logical operator (e.g. "!" or "==" etc) + legal = checkLastToken( + null, + TokenKind.Separator, + TokenKind.StartIndex, + TokenKind.StartGroup, + TokenKind.StartParameters, + TokenKind.LogicalOperator + ) + } } } - if (!legal) { return Token(TokenKind.Unexpected, rawValue, index) } diff --git a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/utils/ExpressionJsonUtil.kt b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/utils/ExpressionJsonUtil.kt index c2060839ba7..95f5f259b60 100644 --- a/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/utils/ExpressionJsonUtil.kt +++ b/src/backend/ci/core/common/common-expression/src/main/kotlin/com/tencent/devops/common/expression/utils/ExpressionJsonUtil.kt @@ -45,5 +45,9 @@ object ExpressionJsonUtil { return objectMapper.readTree(jsonReader) } + fun write(node: JsonNode): String { + return objectMapper.writeValueAsString(node) + } + fun getObjectMapper() = objectMapper } diff --git a/src/backend/ci/core/common/common-expression/src/test/kotlin/com/tencent/devops/common/expression/expression/ExpressionParserTest.kt b/src/backend/ci/core/common/common-expression/src/test/kotlin/com/tencent/devops/common/expression/expression/ExpressionParserTest.kt index a78b6f81980..17ad9d05165 100644 --- a/src/backend/ci/core/common/common-expression/src/test/kotlin/com/tencent/devops/common/expression/expression/ExpressionParserTest.kt +++ b/src/backend/ci/core/common/common-expression/src/test/kotlin/com/tencent/devops/common/expression/expression/ExpressionParserTest.kt @@ -27,6 +27,7 @@ package com.tencent.devops.common.expression.expression +import com.tencent.devops.common.api.util.JsonUtil import com.tencent.devops.common.expression.ExecutionContext import com.tencent.devops.common.expression.ExpressionParseException import com.tencent.devops.common.expression.ExpressionParser @@ -67,7 +68,7 @@ class ExpressionParserTest { "jobs.build.0.steps.runStep.outputs.myoutput" to "build_0", "jobs.build.2.steps.runStep.outputs.myoutput" to "build_1", "depends.job1.outputs.matrix_include" to - """[{"service":"api","var1":"b","var3":"yyy"},{"service":"c","cpu":"zzz"}]""", + """[{"service":"api","var1":"b","var3":"yyy"},{"service":"c","cpu":"zzz"}]""", "project.name.chinese" to "蓝盾项目" ) val expected = listOf( @@ -113,6 +114,146 @@ class ExpressionParserTest { println(ExpressionParser.evaluateByMap("variables.pipeline_id2", variables, true) ?: "variables.pipeline_id2") } + @DisplayName("测试流水线变量中对象的转换") + @Test + fun variablesObjectConvert() { +// val variablesWithError = mapOf( +// "matrix.power" to "{url=cn.com, project=p-xxx}", +// "matrix.power.url" to "cn.com", +// "matrix.power.project" to "p-xxx" +// ) + val variables = mapOf( + "matrix.power" to "{ \"url\" : \"cn.com\", \"project\": \"p-xxx\" }", + "matrix.power.url" to "cn.com", + "matrix.power.project" to "p-xxx", + "name" to "John Doe", + "age" to "30", + "address.street" to "123 Main St", + "address.city" to "New York", + "address.state" to "NY", + "address.coordinates.latitude" to "40.7128", + "address.coordinates.longitude" to "-74.006", + "address.coordinates" to "{\"latitude\":40.7128,\"longitude\":-74.006}", + "address" to "{\"street\":\"123 Main St\",\"city\":\"New York\",\"state\":\"NY\",\"coordinates\":{\"latitude\":40.7128,\"longitude\":-74.006}}", + "phoneNumbers.0.type" to "home", + "phoneNumbers.0.number" to "555-1234", + "phoneNumbers.0" to "{\"type\":\"home\",\"number\":\"555-1234\"}", + "phoneNumbers.1.type" to "work", + "phoneNumbers.1.number" to "555-5678", + "phoneNumbers.1.extensions.0" to "100", + "phoneNumbers.1.extensions.1" to "101", + "phoneNumbers.1.extensions.2" to "102", + "phoneNumbers.1.extensions" to "[100,101,102]", + "phoneNumbers.1" to "{\"type\":\"work\",\"number\":\"555-5678\",\"extensions\":[100,101,102]}", + "phoneNumbers" to "[{\"type\":\"home\",\"number\":\"555-1234\"},{\"type\":\"work\",\"number\":\"555-5678\",\"extensions\":[100,101,102]}]", + "friends.0.name" to "Jane Smith", + "friends.0.age" to "28", + "friends.0.address.street" to "456 Elm St", + "friends.0.address.city" to "San Francisco", + "friends.0.address.state" to "CA", + "friends.0.address.coordinates.latitude" to "37.7749", + "friends.0.address.coordinates.longitude" to "-122.4194", + "friends.0.address.coordinates" to "{\"latitude\":37.7749,\"longitude\":-122.4194}", + "friends.0.address" to "{\"street\":\"456 Elm St\",\"city\":\"San Francisco\",\"state\":\"CA\",\"coordinates\":{\"latitude\":37.7749,\"longitude\":-122.4194}}", + "friends.0.phoneNumbers.0.type" to "home", + "friends.0.phoneNumbers.0.number" to "555-9876", + "friends.0.phoneNumbers.0" to "{\"type\":\"home\",\"number\":\"555-9876\"}", + "friends.0.phoneNumbers.1.type" to "work", + "friends.0.phoneNumbers.1.number" to "555-5432", + "friends.0.phoneNumbers.1" to "{\"type\":\"work\",\"umber\":\"555-5432\"}", + "friends.0.phoneNumbers" to "[{\"type\":\"home\",\"number\":\"555-9876\"},{\"type\":\"work\",\"number\":\"555-5432\"}]", + "friends.0" to "{\"name\":\"Jane Smith\",\"age\":28,\"address\":{\"street\":\"456 Elm St\",\"city\":\"San Francisco\",\"state\":\"CA\",\"coordinates\":{\"latitude\":37.7749,\"longitude\":-122.4194}},\"phoneNumbers\":[{\"type\":\"home\",\"number\":\"555-9876\"},{\"type\":\"work\",\"number\":\"555-5432\"}]}", + "friends.1.name" to "Bob Johnson", + "friends.1.age" to "32", + "friends.1.address.street" to "789 Oak St", + "friends.1.address.city" to "Chicago", + "friends.1.address.state" to "IL", + "friends.1.address.coordinates.latitude" to "41.8781", + "friends.1.address.coordinates.longitude" to "-87.6298", + "friends.1.address.coordinates" to "{\"latitude\":41.8781,\"longitude\":-87.6298}", + "friends.1.address" to "{\"street\":\"789 Oak St\",\"city\":\"Chicago\",\"state\":\"IL\",\"coordinates\":{\"latitude\":41.8781,\"longitude\":-87.6298}}", + "friends.1.phoneNumbers.0.type" to "home", + "friends.1.phoneNumbers.0.number" to "555-1111", + "friends.1.phoneNumbers.0" to "{\"type\":\"home\",\"number\":\"555-1111\"}", + "friends.1.phoneNumbers.1.type" to "work", + "friends.1.phoneNumbers.1.number" to "555-2222", + "friends.1.phoneNumbers.1.extensions.0" to "200", + "friends.1.phoneNumbers.1.extensions.1" to "201", + "friends.1.phoneNumbers.1.extensions" to "[200,201]", + "friends.1.phoneNumbers.1" to "{\"type\":\"work\",\"number\":\"555-2222\",\"extensions\":[200,201]}", + "friends.1.phoneNumbers" to "[{\"type\":\"home\",\"number\":\"555-1111\"},{\"type\":\"work\",\"number\":\"555-2222\",\"extensions\":[200,201]}]", + "friends.1.friends.0.name" to "Alice Brown", + "friends.1.friends.0.age" to "29", + "friends.1.friends.0.address.street" to "321 Pine St", + "friends.1.friends.0.address.city" to "Seattle", + "friends.1.friends.0.address.state" to "WA", + "friends.1.friends.0.address.coordinates.latitude" to "47.6062", + "friends.1.friends.0.address.coordinates.longitude" to "-122.3321", + "friends.1.friends.0.address.coordinates" to "{\"latitude\":47.6062,\"longitude\":-122.3321}", + "friends.1.friends.0.address" to "{\"street\":\"321 Pine St\",\"city\":\"Seattle\",\"state\":\"WA\",\"coordinates\":{\"latitude\":47.6062,\"longitude\":-122.3321}}", + "friends.1.friends.0" to "{\"name\":\"Alice Brown\",\"age\":29,\"address\":{\"street\":\"321 Pine St\",\"city\":\"Seattle\",\"state\":\"WA\",\"coordinates\":{\"latitude\":47.6062,\"longitude\":-122.3321}}}", + "friends.1.friends.1.name" to "Eve Green", + "friends.1.friends.1.age" to "31", + "friends.1.friends.1.address.street" to "654 Cedar St", + "friends.1.friends.1.address.city" to "Austin", + "friends.1.friends.1.address.state" to "TX", + "friends.1.friends.1.address.coordinates.latitude" to "30.2672", + "friends.1.friends.1.address.coordinates.longitude" to "-97.7431", + "friends.1.friends.1.address.coordinates" to "{\"latitude\":30.2672,\"longitude\":-97.7431}", + "friends.1.friends.1.address" to "{\"street\":\"654 Cedar St\",\"city\":\"Austin\",\"state\":\"TX\",\"coordinates\":{\"latitude\":30.2672,\"longitude\":-97.7431}}", + "friends.1.friends.1" to "{\"name\":\"Eve Green\",\"age\":31,\"address\":{\"street\":\"654 Cedar St\",\"city\":\"Austin\",\"state\":\"TX\",\"coordinates\":{\"latitude\":30.2672,\"longitude\":-97.7431}}}", + "friends.1.friends" to "[{\"name\":\"Alice Brown\",\"age\":29,\"address\":{\"street\":\"321 Pine St\",\"city\":\"Seattle\",\"state\":\"WA\",\"coordinates\":{\"latitude\":47.6062,\"longitude\":-122.3321}}},{\"name\":\"Eve Green\",\"age\":31,\"address\":{\"street\":\"654 Cedar St\",\"city\":\"Austin\",\"state\":\"TX\",\"coordinates\":{\"latitude\":30.2672,\"longitude\":-97.7431}}}]", + "friends.1" to "{\"name\":\"Bob Johnson\",\"age\":32,\"address\":{\"street\":\"789 Oak St\",\"city\":\"Chicago\",\"state\":\"IL\",\"coordinates\":{\"latitude\":41.8781,\"longitude\":-87.6298}},\"phoneNumbers\":[{\"type\":\"home\",\"number\":\"555-1111\"},{\"type\":\"work\",\"number\":\"555-2222\",\"extensions\":[200,201]}],\"friends\":[{\"name\":\"Alice Brown\",\"age\":29,\"address\":{\"street\":\"321 Pine St\",\"city\":\"Seattle\",\"state\":\"WA\",\"coordinates\":{\"latitude\":47.6062,\"longitude\":-122.3321}}},{\"name\":\"Eve Green\",\"age\":31,\"address\":{\"street\":\"654 Cedar St\",\"city\":\"Austin\",\"state\":\"TX\",\"coordinates\":{\"latitude\":30.2672,\"longitude\":-97.7431}}}]}", + "friends" to "[{\"name\":\"Jane Smith\",\"age\":28,\"address\":{\"street\":\"456 Elm St\",\"city\":\"San Francisco\",\"state\":\"CA\",\"coordinates\":{\"latitude\":37.7749,\"longitude\":-122.4194}},\"phoneNumbers\":[{\"type\":\"home\",\"number\":\"555-9876\"},{\"type\":\"work\",\"number\":\"555-5432\"}]},{\"name\":\"Bob Johnson\",\"age\":32,\"address\":{\"street\":\"789 Oak St\",\"city\":\"Chicago\",\"state\":\"IL\",\"coordinates\":{\"latitude\":41.8781,\"longitude\":-87.6298}},\"phoneNumbers\":[{\"type\":\"home\",\"number\":\"555-1111\"},{\"type\":\"work\",\"number\":\"555-2222\",\"extensions\":[200,201]}],\"friends\":[{\"name\":\"Alice Brown\",\"age\":29,\"address\":{\"street\":\"321 Pine St\",\"city\":\"Seattle\",\"state\":\"WA\",\"coordinates\":{\"latitude\":47.6062,\"longitude\":-122.3321}}},{\"name\":\"Eve Green\",\"age\":31,\"address\":{\"street\":\"654 Cedar St\",\"city\":\"Austin\",\"state\":\"TX\",\"coordinates\":{\"latitude\":30.2672,\"longitude\":-97.7431}}}]}]", + "ep_rsp_result" to "true" + ) + val jsonKeys = setOf( + "matrix.power", + "address.coordinates", + "address", + "phoneNumbers.0", + "phoneNumbers.1", + "phoneNumbers.1.extensions", + "phoneNumbers", + "friends.0.address.coordinates", + "friends.0.address", + "friends.0.phoneNumbers.0", + "friends.0.phoneNumbers.1", + "friends.0.phoneNumbers", + "friends.0", + "friends.1.address.coordinates", + "friends.1.address", + "friends.1.phoneNumbers.0", + "friends.1.phoneNumbers.1.extensions", + "friends.1.phoneNumbers.1", + "friends.1.phoneNumbers", + "friends.1.friends.0.address.coordinates", + "friends.1.friends.0.address", + "friends.1.friends.0", + "friends.1.friends.1.address.coordinates", + "friends.1.friends.1.address", + "friends.1.friends.1", + "friends.1.friends", + "friends.1", + "friends" + ) + variables.forEach { (k, v) -> + if (k in jsonKeys) { + Assertions.assertEquals( + JsonUtil.getObjectMapper().readTree(v), + JsonUtil.getObjectMapper().readTree( + ExpressionParser.evaluateByMap(k, variables, true).toString() + ) + ) + return@forEach + } + Assertions.assertEquals(v, ExpressionParser.evaluateByMap(k, variables, true)) + } +// assertThrows { +// ExpressionParser.evaluateByMap("matrix.power.url=='cn.com'", variablesWithError, true) +// } + } + @DisplayName("测试解析文字") @Test fun literalsTest() { @@ -346,7 +487,7 @@ class ExpressionParserTest { @ValueSource( strings = [ "1 => fromJson('{\"include\":[{\"project\":\"foo\",\"config\":\"Debug\"},{\"project\":\"bar\"," + - "\"config\":\"Release\"}]}')", + "\"config\":\"Release\"}]}')", "2 => fromJson(funcTest.json.strJson)", "3 => fromJson(funcTest.json.boolJson)", "4 => fromJson(funcTest.json.numJson)" diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/ModelCreate.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/ModelCreate.kt index 0f19dcd012d..7accac9abfc 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/ModelCreate.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/ModelCreate.kt @@ -35,20 +35,22 @@ import com.tencent.devops.common.pipeline.Model import com.tencent.devops.common.pipeline.container.Stage import com.tencent.devops.common.pipeline.container.TriggerContainer import com.tencent.devops.common.pipeline.pojo.BuildFormProperty +import com.tencent.devops.common.pipeline.pojo.PipelineModelAndSetting import com.tencent.devops.common.pipeline.pojo.element.Element +import com.tencent.devops.common.pipeline.pojo.element.market.MarketBuildAtomElement import com.tencent.devops.common.pipeline.pojo.element.trigger.ManualTriggerElement +import com.tencent.devops.common.pipeline.pojo.setting.PipelineRunLockType +import com.tencent.devops.common.pipeline.pojo.setting.PipelineSetting import com.tencent.devops.common.web.utils.I18nUtil import com.tencent.devops.process.api.user.UserPipelineGroupResource import com.tencent.devops.process.engine.common.VMUtils import com.tencent.devops.process.pojo.classify.PipelineGroup import com.tencent.devops.process.pojo.classify.PipelineGroupCreate import com.tencent.devops.process.pojo.classify.PipelineLabelCreate -import com.tencent.devops.common.pipeline.pojo.PipelineModelAndSetting -import com.tencent.devops.common.pipeline.pojo.setting.PipelineRunLockType -import com.tencent.devops.common.pipeline.pojo.setting.PipelineSetting import com.tencent.devops.process.yaml.creator.inner.ModelCreateEvent import com.tencent.devops.process.yaml.pojo.QualityElementInfo import com.tencent.devops.process.yaml.v2.models.ScriptBuildYaml +import com.tencent.devops.store.api.atom.ServiceMarketAtomResource import java.util.concurrent.TimeUnit import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired @@ -142,16 +144,18 @@ class ModelCreate @Autowired constructor( ) ) } + val model = Model( + name = modelName, + desc = "", + stages = stageList, + labels = labelList, + instanceFromTemplate = false, + pipelineCreator = event.userId + ) + modelInstallAtom(model, event.projectCode, event.elementInstallUserId) return PipelineModelAndSetting( - model = Model( - name = modelName, - desc = "", - stages = stageList, - labels = labelList, - instanceFromTemplate = false, - pipelineCreator = event.userId - ), + model = model, setting = PipelineSetting( concurrencyGroup = yaml.concurrency?.group, // Cancel-In-Progress 配置group后默认为true @@ -172,6 +176,27 @@ class ModelCreate @Autowired constructor( ) } + @Suppress("NestedBlockDepth") + private fun modelInstallAtom(model: Model, projectId: String, elementInstallUserId: String) { + val install = client.get(ServiceMarketAtomResource::class).getProjectElements( + projectCode = projectId + ).data?.keys ?: emptyList() + model.stages.forEach { stage -> + stage.containers.forEach { container -> + container.elements.forEach { element -> + if (element is MarketBuildAtomElement && element.getAtomCode() !in install) { + ModelCommon.installMarketAtom( + client = client, + projectCode = projectId, + userId = elementInstallUserId, + atomCode = element.getAtomCode() + ) + } + } + } + } + } + @Suppress("NestedBlockDepth") private fun preparePipelineLabels( event: ModelCreateEvent, diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/ModelElement.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/ModelElement.kt index 0407c6673ac..d34a467f02a 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/ModelElement.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/creator/ModelElement.kt @@ -118,14 +118,6 @@ class ModelElement @Autowired(required = false) constructor( element.canRetry = false element.customEnv = ModelCommon.getCustomEnv(step.env) elementList.add(element) - if (element is MarketBuildAtomElement) { - ModelCommon.installMarketAtom( - client = client, - projectCode = event.projectCode, - userId = event.elementInstallUserId, - atomCode = element.getAtomCode() - ) - } } } @@ -177,6 +169,7 @@ class ModelElement @Autowired(required = false) constructor( scriptType = BuildScriptType.BAT, script = step.run ) + else -> linux } } diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/ContainerTransfer.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/ContainerTransfer.kt index f048df51709..b99fb572a33 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/ContainerTransfer.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/ContainerTransfer.kt @@ -288,8 +288,8 @@ class ContainerTransfer @Autowired(required = false) constructor( if (hwSpec != null && buildType != null) { kotlin.run { val res = transferCache.getDockerResource(userId, projectId, buildType) - // hwSpec为0时为特殊值，表示默认配置 - if (res?.default == hwSpec || hwSpec == "0") { + // hwSpec为0和1时为特殊值，表示默认配置Basic + if (res?.default == hwSpec || hwSpec == "0" || hwSpec == "1") { hwSpec = null return@run } @@ -309,9 +309,9 @@ class ContainerTransfer @Autowired(required = false) constructor( } private fun makeJobTimeout(controlOption: JobControlOption?): String? { - return controlOption?.timeoutVar.nullIfDefault( + return (controlOption?.timeoutVar ?: controlOption?.timeout?.toString()).nullIfDefault( VariableDefault.DEFAULT_JOB_MAX_RUNNING_MINUTES.toString() - ) ?: controlOption?.timeout.nullIfDefault(VariableDefault.DEFAULT_JOB_MAX_RUNNING_MINUTES)?.toString() + ) } @Suppress("UNCHECKED_CAST") @@ -426,8 +426,8 @@ class ContainerTransfer @Autowired(required = false) constructor( null }, timeoutMinutes = if (resource.queueEnable) { - resource.timeoutVar.nullIfDefault(DEFAULT_MUTEX_TIMEOUT_MINUTES.toString()) - ?: resource.timeout.nullIfDefault(DEFAULT_MUTEX_TIMEOUT_MINUTES)?.toString() + (resource.timeoutVar ?: resource.timeout.toString()) + .nullIfDefault(DEFAULT_MUTEX_TIMEOUT_MINUTES.toString()) } else { null } diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/ElementTransfer.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/ElementTransfer.kt index c2c4f0d65ec..e1445c581e9 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/ElementTransfer.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/ElementTransfer.kt @@ -29,6 +29,7 @@ package com.tencent.devops.process.yaml.transfer import com.fasterxml.jackson.core.type.TypeReference import com.tencent.devops.common.api.enums.ScmType +import com.tencent.devops.common.api.enums.TriggerRepositoryType import com.tencent.devops.common.api.util.JsonUtil import com.tencent.devops.common.client.Client import com.tencent.devops.common.pipeline.container.Container @@ -151,6 +152,19 @@ class ElementTransfer @Autowired(required = false) constructor( else -> return@m "" } } + // ui->code,repositoryType为null时,repoType才需要在code模式下展示 + val (repoType, repoHashId, repoName) = when { + element.repositoryType == TriggerRepositoryType.ID && !element.repoHashId.isNullOrBlank() -> + Triple(null, element.repoHashId, null) + + element.repositoryType == TriggerRepositoryType.NAME && !element.repoName.isNullOrBlank() -> + Triple(null, null, element.repoName) + + element.repositoryType == TriggerRepositoryType.SELF -> + Triple(null, null, null) + + else -> Triple(TriggerRepositoryType.NONE.name, null, null) + } schedules.add( SchedulesRule( name = element.name, @@ -160,8 +174,9 @@ class ElementTransfer @Autowired(required = false) constructor( } else { element.advanceExpression }, - repoId = element.repoHashId, - repoName = element.repoName, + repoType = repoType, + repoId = repoHashId, + repoName = repoName, branches = element.branches, always = (element.noScm != true).nullIfDefault(false), enable = element.isElementEnable().nullIfDefault(true) @@ -207,7 +222,7 @@ class ElementTransfer @Autowired(required = false) constructor( elements = gitElement, projectId = projectId, aspectWrapper = aspectWrapper, - defaultName = "Git事件触发" + defaultName = "Git" ) res.putAll(gitTrigger.groupBy { ScmType.CODE_GIT }) } @@ -221,7 +236,7 @@ class ElementTransfer @Autowired(required = false) constructor( elements = tGitElement, projectId = projectId, aspectWrapper = aspectWrapper, - defaultName = "TGit事件触发" + defaultName = "TGit" ) res.putAll(gitTrigger.groupBy { ScmType.CODE_TGIT }) } @@ -235,7 +250,7 @@ class ElementTransfer @Autowired(required = false) constructor( elements = githubElement, projectId = projectId, aspectWrapper = aspectWrapper, - defaultName = "GitHub事件触发" + defaultName = "GitHub" ) res.putAll(gitTrigger.groupBy { ScmType.GITHUB }) } @@ -249,7 +264,7 @@ class ElementTransfer @Autowired(required = false) constructor( elements = svnElement, projectId = projectId, aspectWrapper = aspectWrapper, - defaultName = "SVN事件触发" + defaultName = "SVN" ) res.putAll(gitTrigger.groupBy { ScmType.CODE_SVN }) } @@ -263,7 +278,7 @@ class ElementTransfer @Autowired(required = false) constructor( elements = p4Element, projectId = projectId, aspectWrapper = aspectWrapper, - defaultName = "P4事件触发" + defaultName = "P4" ) res.putAll(gitTrigger.groupBy { ScmType.CODE_P4 }) } @@ -277,7 +292,7 @@ class ElementTransfer @Autowired(required = false) constructor( elements = gitlabElement, projectId = projectId, aspectWrapper = aspectWrapper, - defaultName = "Gitlab变更触发" + defaultName = "Gitlab" ) res.putAll(gitTrigger.groupBy { ScmType.CODE_GITLAB }) } @@ -338,7 +353,7 @@ class ElementTransfer @Autowired(required = false) constructor( manualSkip = continueOnError == Step.ContinueOnErrorType.MANUAL_SKIP, timeout = step.timeoutMinutes?.toLongOrNull() ?: VariableDefault.DEFAULT_TASK_TIME_OUT, timeoutVar = step.timeoutMinutes ?: VariableDefault.DEFAULT_TASK_TIME_OUT.toString(), - retryWhenFailed = step.retryTimes != null, + retryWhenFailed = step.retryTimes != null && step.retryTimes > 0, retryCount = step.retryTimes ?: VariableDefault.DEFAULT_RETRY_COUNT, runCondition = runCondition, customCondition = if (runCondition == RunCondition.CUSTOM_CONDITION_MATCH) step.ifFiled else null, @@ -548,9 +563,10 @@ class ElementTransfer @Autowired(required = false) constructor( else -> element.transferYaml(transferCache.getAtomDefaultValue(uses)) }?.apply { this.enable = element.isElementEnable().nullIfDefault(true) - this.timeoutMinutes = element.additionalOptions?.timeoutVar.nullIfDefault( - VariableDefault.DEFAULT_TASK_TIME_OUT.toString() - ) ?: element.additionalOptions?.timeout.nullIfDefault(VariableDefault.DEFAULT_TASK_TIME_OUT)?.toString() + this.timeoutMinutes = + (element.additionalOptions?.timeoutVar ?: element.additionalOptions?.timeout?.toString()).nullIfDefault( + VariableDefault.DEFAULT_TASK_TIME_OUT.toString() + ) this.continueOnError = when { element.additionalOptions?.manualSkip == true -> Step.ContinueOnErrorType.MANUAL_SKIP.alis diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/ModelTransfer.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/ModelTransfer.kt index 9b0bc4883f6..e30a7f45e48 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/ModelTransfer.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/ModelTransfer.kt @@ -283,13 +283,13 @@ class ModelTransfer @Autowired constructor( private fun makeNoticesV3(setting: PipelineSetting): List? { val res = mutableListOf() - setting.successSubscriptionList?.ifEmpty { listOf(setting.successSubscription) }?.forEach { + setting.successSubscriptionList?.ifEmpty { setting.successSubscription?.let { listOf(it) } }?.forEach { if (it.types.isNotEmpty()) { val notice = PacNotices(it, IfType.SUCCESS.name) res.add(prepareYamlGroups(setting.projectId, notice)) } } - setting.failSubscriptionList?.ifEmpty { listOf(setting.failSubscription) }?.forEach { + setting.failSubscriptionList?.ifEmpty { setting.failSubscription?.let { listOf(it) } }?.forEach { if (it.types.isNotEmpty()) { val notice = PacNotices(it, IfType.FAILURE.name) res.add(prepareYamlGroups(setting.projectId, notice)) diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/StageTransfer.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/StageTransfer.kt index 02709b6d50e..2cd0896da97 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/StageTransfer.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/StageTransfer.kt @@ -320,9 +320,9 @@ class StageTransfer @Autowired(required = false) constructor( ) } - private fun maskYamlStageLabel(tags: List?): List { + private fun maskYamlStageLabel(tags: List?): List { if (tags.isNullOrEmpty()) return emptyList() - return tags.map { StageLabel.parseById(it).value } + return tags.filterNotNull().map { StageLabel.parseById(it).value } } private fun getCheckInForStage(stage: Stage): PreStageCheck? { diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/TransferMapper.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/TransferMapper.kt index ef893ff2781..ef96d793522 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/TransferMapper.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/TransferMapper.kt @@ -572,6 +572,7 @@ object TransferMapper { * 2.保留锚点信息 * */ fun mergeYaml(old: String, new: String): String { + if (old.isBlank()) return new val oldE = getYamlFactory().parse(old.reader()).toList() val newL = getYamlFactory().parse(new.reader()).toList() diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/TriggerTransfer.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/TriggerTransfer.kt index 1a222ed0d83..c50f543ae79 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/TriggerTransfer.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/TriggerTransfer.kt @@ -27,6 +27,7 @@ package com.tencent.devops.process.yaml.transfer +import com.tencent.devops.common.api.constant.CommonMessageCode import com.tencent.devops.common.api.enums.RepositoryType import com.tencent.devops.common.api.enums.TriggerRepositoryType import com.tencent.devops.common.client.Client @@ -48,11 +49,14 @@ import com.tencent.devops.common.pipeline.pojo.element.trigger.RemoteTriggerElem import com.tencent.devops.common.pipeline.pojo.element.trigger.TimerTriggerElement import com.tencent.devops.common.pipeline.pojo.element.trigger.enums.CodeEventType import com.tencent.devops.common.pipeline.pojo.element.trigger.enums.PathFilterType +import com.tencent.devops.common.webhook.enums.code.tgit.TGitMrEventAction +import com.tencent.devops.common.webhook.enums.code.tgit.TGitPushActionType import com.tencent.devops.process.yaml.transfer.VariableDefault.nullIfDefault import com.tencent.devops.process.yaml.transfer.aspect.PipelineTransferAspectWrapper import com.tencent.devops.process.yaml.transfer.inner.TransferCreator import com.tencent.devops.process.yaml.transfer.pojo.WebHookTriggerElementChanger import com.tencent.devops.process.yaml.transfer.pojo.YamlTransferInput +import com.tencent.devops.process.yaml.v3.models.on.CustomFilter import com.tencent.devops.process.yaml.v3.models.on.EnableType import com.tencent.devops.process.yaml.v3.models.on.IssueRule import com.tencent.devops.process.yaml.v3.models.on.MrRule @@ -96,9 +100,15 @@ class TriggerTransfer @Autowired(required = false) constructor( pathFilterType = push.pathFilterType?.let { PathFilterType.valueOf(it) } ?: PathFilterType.NamePrefixFilter, eventType = CodeEventType.PUSH, - includePushAction = push.action, + includePushAction = push.action ?: listOf( + TGitPushActionType.PUSH_FILE.value, + TGitPushActionType.NEW_BRANCH.value + ), repositoryType = repositoryType, - repositoryName = triggerOn.repoName + repositoryName = triggerOn.repoName, + enableThirdFilter = !push.custom?.url.isNullOrBlank(), + thirdUrl = push.custom?.url, + thirdSecretToken = push.custom?.credentials ).checkTriggerElementEnable(push.enable).apply { version = "2.*" } @@ -140,10 +150,17 @@ class TriggerTransfer @Autowired(required = false) constructor( enableCheck = mr.reportCommitCheck, pathFilterType = mr.pathFilterType?.let { PathFilterType.valueOf(it) } ?: PathFilterType.NamePrefixFilter, - includeMrAction = mr.action, + includeMrAction = mr.action ?: listOf( + TGitMrEventAction.OPEN.value, + TGitMrEventAction.REOPEN.value, + TGitMrEventAction.PUSH_UPDATE.value + ), eventType = CodeEventType.MERGE_REQUEST, repositoryType = repositoryType, - repositoryName = triggerOn.repoName + repositoryName = triggerOn.repoName, + enableThirdFilter = !mr.custom?.url.isNullOrBlank(), + thirdUrl = mr.custom?.url, + thirdSecretToken = mr.custom?.credentials ).checkTriggerElementEnable(mr.enable).apply { version = "2.*" } @@ -236,14 +253,18 @@ class TriggerTransfer @Autowired(required = false) constructor( CodeEventType.PUSH -> nowExist.push = PushRule( name = git.name.nullIfDefault(defaultName), enable = git.enable.nullIfDefault(true), - branches = git.branchName?.disjoin() ?: emptyList(), + branches = git.branchName?.disjoin(), branchesIgnore = git.excludeBranchName?.disjoin(), paths = git.includePaths?.disjoin(), pathsIgnore = git.excludePaths?.disjoin(), users = git.includeUsers, usersIgnore = git.excludeUsers, pathFilterType = git.pathFilterType?.name.nullIfDefault(PathFilterType.NamePrefixFilter.name), - action = git.includePushAction + action = git.includePushAction, + custom = if (git.enableThirdFilter == true) CustomFilter( + url = git.thirdUrl, + credentials = git.thirdSecretToken + ) else null ) CodeEventType.TAG_PUSH -> nowExist.tag = TagRule( @@ -271,8 +292,17 @@ class TriggerTransfer @Autowired(required = false) constructor( webhookQueue = git.webhookQueue.nullIfDefault(false), reportCommitCheck = git.enableCheck.nullIfDefault(true), pathFilterType = git.pathFilterType?.name.nullIfDefault(PathFilterType.NamePrefixFilter.name), - action = git.includeMrAction + action = git.includeMrAction, + custom = if (git.enableThirdFilter == true) CustomFilter( + url = git.thirdUrl, + credentials = git.thirdSecretToken + ) else null ) + CodeEventType.MERGE_REQUEST_ACCEPT -> + throw PipelineTransferException( + errorCode = CommonMessageCode.MR_ACCEPT_EVENT_NOT_SUPPORT_TRANSFER, + params = arrayOf(git.name) + ) CodeEventType.REVIEW -> nowExist.review = ReviewRule( name = git.name.nullIfDefault(defaultName), @@ -312,14 +342,20 @@ class TriggerTransfer @Autowired(required = false) constructor( pathFilterType = git.pathFilterType?.name.nullIfDefault(PathFilterType.NamePrefixFilter.name) ) - CodeEventType.CHANGE_COMMIT -> nowExist.push = PushRule( - name = git.name.nullIfDefault(defaultName), - enable = git.enable.nullIfDefault(true), - branches = null, - branchesIgnore = null, - paths = git.includePaths?.disjoin(), - pathsIgnore = git.excludePaths?.disjoin() - ) + in CodeEventType.CODE_P4_EVENTS -> { + buildP4TriggerOn( + codeEventType = git.eventType, + triggerOn = nowExist, + rule = PushRule( + name = git.name.nullIfDefault(defaultName), + enable = git.enable.nullIfDefault(true), + branches = null, + branchesIgnore = null, + paths = git.includePaths?.disjoin(), + pathsIgnore = git.excludePaths?.disjoin() + ) + ) + } CodeEventType.PULL_REQUEST -> nowExist.mr = MrRule( name = git.name.nullIfDefault(defaultName), @@ -365,7 +401,10 @@ class TriggerTransfer @Autowired(required = false) constructor( pathFilterType = push.pathFilterType?.let { PathFilterType.valueOf(it) } ?: PathFilterType.NamePrefixFilter, eventType = CodeEventType.PUSH, - includeMrAction = push.action, + includeMrAction = push.action ?: listOf( + TGitPushActionType.PUSH_FILE.value, + TGitPushActionType.NEW_BRANCH.value + ), repositoryType = repositoryType, repositoryName = triggerOn.repoName ) @@ -417,7 +456,11 @@ class TriggerTransfer @Autowired(required = false) constructor( enableCheck = mr.reportCommitCheck, pathFilterType = mr.pathFilterType?.let { PathFilterType.valueOf(it) } ?: PathFilterType.NamePrefixFilter, - includeMrAction = mr.action, + includeMrAction = mr.action ?: listOf( + TGitMrEventAction.OPEN.value, + TGitMrEventAction.REOPEN.value, + TGitMrEventAction.PUSH_UPDATE.value + ), eventType = CodeEventType.MERGE_REQUEST, repositoryType = repositoryType, repositoryName = triggerOn.repoName @@ -633,25 +676,21 @@ class TriggerTransfer @Autowired(required = false) constructor( @Suppress("ComplexMethod") fun yaml2TriggerP4(triggerOn: TriggerOn, elementQueue: MutableList) { - val repositoryType = if (triggerOn.repoName.isNullOrBlank()) { - TriggerRepositoryType.SELF - } else { - TriggerRepositoryType.NAME - } - triggerOn.push?.let { push -> - elementQueue.add( - CodeP4WebHookTriggerElement( - name = push.name ?: "P4事件触发", - data = CodeP4WebHookTriggerData( - input = CodeP4WebHookTriggerInput( - includePaths = push.paths.nonEmptyOrNull()?.join(), - excludePaths = push.pathsIgnore.nonEmptyOrNull()?.join(), - eventType = CodeEventType.CHANGE_COMMIT, - repositoryType = repositoryType, - repositoryName = triggerOn.repoName - ) - ) - ).checkTriggerElementEnable(push.enable).apply { version = "2.*" } + with(triggerOn) { + val repositoryType = if (repoName.isNullOrBlank()) { + TriggerRepositoryType.SELF + } else { + TriggerRepositoryType.NAME + } + elementQueue.addAll( + listOfNotNull( + buildP4TriggerElement(push, CodeEventType.CHANGE_COMMIT, repositoryType, repoName), + buildP4TriggerElement(changeCommit, CodeEventType.CHANGE_COMMIT, repositoryType, repoName), + buildP4TriggerElement(changeContent, CodeEventType.CHANGE_CONTENT, repositoryType, repoName), + buildP4TriggerElement(changeSubmit, CodeEventType.CHANGE_SUBMIT, repositoryType, repoName), + buildP4TriggerElement(shelveCommit, CodeEventType.SHELVE_COMMIT, repositoryType, repoName), + buildP4TriggerElement(shelveSubmit, CodeEventType.SHELVE_SUBMIT, repositoryType, repoName) + ) ) } } @@ -675,12 +714,15 @@ class TriggerTransfer @Autowired(required = false) constructor( schedule.forEach { timer -> val repositoryType = when { !timer.repoId.isNullOrBlank() -> - RepositoryType.ID + TriggerRepositoryType.ID !timer.repoName.isNullOrBlank() -> - RepositoryType.NAME + TriggerRepositoryType.NAME - else -> null + timer.repoType == TriggerRepositoryType.NONE.name -> + null + // code -> ui,默认监听PAC代码库 + else -> TriggerRepositoryType.SELF } elementQueue.add( TimerTriggerElement( @@ -723,7 +765,7 @@ class TriggerTransfer @Autowired(required = false) constructor( triggerOn.push?.let { push -> elementQueue.add( CodeGitlabWebHookTriggerElement( - name = push.name ?: "Gitlab变更触发", + name = push.name ?: "Gitlab事件触发", branchName = push.branches.nonEmptyOrNull()?.join(), excludeBranchName = push.branchesIgnore.nonEmptyOrNull()?.join(), includePaths = push.paths.nonEmptyOrNull()?.join(), @@ -733,10 +775,15 @@ class TriggerTransfer @Autowired(required = false) constructor( pathFilterType = push.pathFilterType?.let { PathFilterType.valueOf(it) } ?: PathFilterType.NamePrefixFilter, eventType = CodeEventType.PUSH, - // todo action + includeMrAction = push.action ?: listOf( + TGitPushActionType.PUSH_FILE.value, + TGitPushActionType.NEW_BRANCH.value + ), repositoryType = repositoryType, repositoryName = triggerOn.repoName - ).checkTriggerElementEnable(push.enable) + ).checkTriggerElementEnable(push.enable).apply { + version = "2.*" + } ) } @@ -758,7 +805,7 @@ class TriggerTransfer @Autowired(required = false) constructor( triggerOn.mr?.let { mr -> elementQueue.add( CodeGitlabWebHookTriggerElement( - name = mr.name ?: "Gitlab变更触发", + name = mr.name ?: "Gitlab事件触发", branchName = mr.targetBranches.nonEmptyOrNull()?.join(), excludeBranchName = mr.targetBranchesIgnore.nonEmptyOrNull()?.join(), includeSourceBranchName = mr.sourceBranches.nonEmptyOrNull()?.join(), @@ -770,11 +817,17 @@ class TriggerTransfer @Autowired(required = false) constructor( block = mr.blockMr, pathFilterType = mr.pathFilterType?.let { PathFilterType.valueOf(it) } ?: PathFilterType.NamePrefixFilter, - // todo action + includeMrAction = mr.action ?: listOf( + TGitMrEventAction.OPEN.value, + TGitMrEventAction.REOPEN.value, + TGitMrEventAction.PUSH_UPDATE.value + ), eventType = CodeEventType.MERGE_REQUEST, repositoryType = repositoryType, repositoryName = triggerOn.repoName - ).checkTriggerElementEnable(mr.enable) + ).checkTriggerElementEnable(mr.enable).apply { + version = "2.*" + } ) } } @@ -792,4 +845,39 @@ class TriggerTransfer @Autowired(required = false) constructor( private fun String.disjoin() = this.split(",") private fun List?.nonEmptyOrNull() = this?.ifEmpty { null } + + private fun buildP4TriggerElement( + rule: PushRule?, + eventType: CodeEventType, + repositoryType: TriggerRepositoryType, + repoName: String? + ): Element? { + return rule?.let { + CodeP4WebHookTriggerElement( + name = rule.name ?: "P4事件触发", + data = CodeP4WebHookTriggerData( + input = CodeP4WebHookTriggerInput( + includePaths = rule.paths.nonEmptyOrNull()?.join(), + excludePaths = rule.pathsIgnore.nonEmptyOrNull()?.join(), + eventType = eventType, + repositoryType = repositoryType, + repositoryName = repoName + ) + ) + ).checkTriggerElementEnable(rule.enable).apply { + // P4触发器(v2)仅支持CHANGE_COMMIT事件 + version = if (eventType == CodeEventType.CHANGE_COMMIT) "2.*" else "1.*" + } + } + } + + fun buildP4TriggerOn(codeEventType: CodeEventType?, triggerOn: TriggerOn, rule: PushRule) { + when (codeEventType) { + CodeEventType.CHANGE_COMMIT -> triggerOn.changeCommit = rule + CodeEventType.CHANGE_SUBMIT -> triggerOn.changeSubmit = rule + CodeEventType.CHANGE_CONTENT -> triggerOn.changeContent = rule + CodeEventType.SHELVE_COMMIT -> triggerOn.shelveCommit = rule + CodeEventType.SHELVE_SUBMIT -> triggerOn.shelveSubmit = rule + } + } } diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/VariableTransfer.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/VariableTransfer.kt index a7fb91f03c3..78588851f4f 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/VariableTransfer.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/VariableTransfer.kt @@ -46,12 +46,11 @@ import com.tencent.devops.process.yaml.v3.models.VariablePropOption import com.tencent.devops.process.yaml.v3.models.VariablePropType import com.tencent.devops.process.yaml.v3.models.VariableProps import org.slf4j.LoggerFactory -import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Component @Component @Suppress("ComplexMethod") -class VariableTransfer @Autowired constructor() { +class VariableTransfer { companion object { private val logger = LoggerFactory.getLogger(VariableTransfer::class.java) @@ -77,7 +76,7 @@ class VariableTransfer @Autowired constructor() { it.type == BuildFormPropertyType.ENUM -> VariableProps( type = VariablePropType.SELECTOR.value, options = it.options?.map { form -> - VariablePropOption(id = form.value, label = form.key) + VariablePropOption(id = form.key, label = form.value) }, payload = it.payload ) @@ -97,7 +96,7 @@ class VariableTransfer @Autowired constructor() { it.type == BuildFormPropertyType.MULTIPLE -> VariableProps( type = VariablePropType.CHECKBOX.value, options = it.options?.map { form -> - VariablePropOption(id = form.value, label = form.key) + VariablePropOption(id = form.key, label = form.value) }, payload = it.payload ) @@ -215,10 +214,7 @@ class VariableTransfer @Autowired constructor() { else -> variable.value ?: "" }, options = variable.props?.options?.map { - BuildFormValue( - key = it.label ?: it.id.toString(), - value = it.id.toString() - ) + BuildFormValue(key = it.id.toString(), value = it.label ?: it.id.toString()) }, desc = variable.props?.description, repoHashId = variable.props?.repoHashId, @@ -236,7 +232,8 @@ class VariableTransfer @Autowired constructor() { readOnly = if (variable.const == true) true else { variable.readonly ?: false }, - valueNotEmpty = variable.props?.required ?: false + valueNotEmpty = variable.props?.required ?: false, + payload = variable.props?.payload ) ) } diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/pojo/WebHookTriggerElementChanger.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/pojo/WebHookTriggerElementChanger.kt index aaeef46ac6d..b1ddc853ee4 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/pojo/WebHookTriggerElementChanger.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/transfer/pojo/WebHookTriggerElementChanger.kt @@ -242,6 +242,8 @@ data class WebHookTriggerElementChanger( excludeTagName = data.excludeTagName, excludeSourceBranchName = data.excludeSourceBranchName, includeSourceBranchName = data.includeSourceBranchName, + includeMrAction = data.includeMrAction, + includePushAction = data.includePushAction, enable = data.isElementEnable() ) } diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/parsers/template/YamlObjects.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/parsers/template/YamlObjects.kt index 01ba38a0d3b..ba9175d3452 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/parsers/template/YamlObjects.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v2/parsers/template/YamlObjects.kt @@ -259,7 +259,8 @@ object YamlObjects { null } else { transValue>(fromPath, "mounts", optionsMap["mounts"]) - } + }, + privileged = getNullValue("privileged", optionsMap)?.toBoolean() ) }, imagePullPolicy = getNullValue(key = "image-pull-policy", map = containerMap) diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/PreTemplateScriptBuildYamlV3Parser.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/PreTemplateScriptBuildYamlV3Parser.kt index b74787c9276..3a976308be1 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/PreTemplateScriptBuildYamlV3Parser.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/PreTemplateScriptBuildYamlV3Parser.kt @@ -32,15 +32,18 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.annotation.JsonInclude import com.fasterxml.jackson.annotation.JsonProperty import com.fasterxml.jackson.core.type.TypeReference +import com.tencent.devops.common.api.constant.CommonMessageCode.YAML_NOT_VALID import com.tencent.devops.common.api.enums.ScmType import com.tencent.devops.common.api.util.JsonUtil import com.tencent.devops.common.pipeline.pojo.transfer.Resources import com.tencent.devops.process.yaml.pojo.YamlVersion +import com.tencent.devops.process.yaml.transfer.PipelineTransferException import com.tencent.devops.process.yaml.v3.models.job.Job import com.tencent.devops.process.yaml.v3.models.on.PreTriggerOnV3 import com.tencent.devops.process.yaml.v3.models.on.TriggerOn import com.tencent.devops.process.yaml.v3.models.stage.Stage import com.tencent.devops.process.yaml.v3.utils.ScriptYmlUtils +import org.slf4j.LoggerFactory @JsonInclude(JsonInclude.Include.NON_NULL) @JsonIgnoreProperties(ignoreUnknown = true) @@ -67,6 +70,9 @@ data class PreTemplateScriptBuildYamlV3Parser( @JsonProperty("custom-build-num") override var customBuildNum: String? = null ) : IPreTemplateScriptBuildYamlParser, ITemplateFilter { + companion object { + private val logger = LoggerFactory.getLogger(PreTemplateScriptBuildYamlV3Parser::class.java) + } init { version = YamlVersion.V3_0.tag @@ -94,7 +100,15 @@ data class PreTemplateScriptBuildYamlV3Parser( val transferData: YamlTransferData = YamlTransferData() override fun replaceTemplate(f: (param: ITemplateFilter) -> PreScriptBuildYamlIParser) { - preYaml = f(this) as PreScriptBuildYamlV3Parser + kotlin.runCatching { + preYaml = f(this) as PreScriptBuildYamlV3Parser + }.onFailure { error -> + logger.warn("replaceTemplate error", error) + throw PipelineTransferException( + YAML_NOT_VALID, + arrayOf(error.message ?: "unknown error") + ) + } } override fun formatVariables(): Map { diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/Variable.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/Variable.kt index 5450a924664..69c4034ae3d 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/Variable.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/Variable.kt @@ -89,8 +89,7 @@ data class TemplateVariable(private val list: List) : List by * Variable 属性变量 * @param label 可选, 预定义下拉可选值的字段 * @param type 类型 - * @param options 下拉列表可选值，和 datasource 二选一 - * @param datasource 下拉列表数据源，和 values 二选一 + * @param options 下拉列表可选值 * @param multiple 是否允许多选，缺省时为 false（type=selector时生效） * @param description 可选，描述 * @param required 可选，是否必填 @@ -101,7 +100,6 @@ data class VariableProps( var label: String? = null, val type: String? = null, val options: List? = null, - val datasource: VariableDatasource? = null, var description: String? = null, val multiple: Boolean? = null, var required: Boolean? = null, diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/image/PoolType.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/image/PoolType.kt index 14c42e48d71..2180c19db7e 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/image/PoolType.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/image/PoolType.kt @@ -60,7 +60,7 @@ enum class PoolType { } override fun validatePool(pool: Pool) { - if (null == pool.container) { + if (null == pool.container && pool.image == null) { throw OperationException("当pool.type=$this, container参数不能为空") } } diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/on/MrRule.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/on/MrRule.kt index a2ffd9ea28c..63acc584f17 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/on/MrRule.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/on/MrRule.kt @@ -84,5 +84,9 @@ data class MrRule( @get:Schema(title = "path-filter-type") @JsonProperty("path-filter-type") - val pathFilterType: String? = null + val pathFilterType: String? = null, + + @get:Schema(title = "custom-filter") + @JsonProperty("custom-filter") + val custom: CustomFilter? = null ) diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/on/PreTriggerOnV3.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/on/PreTriggerOnV3.kt index 335db58abf5..40675756115 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/on/PreTriggerOnV3.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/on/PreTriggerOnV3.kt @@ -53,7 +53,22 @@ data class PreTriggerOnV3( override val repoHook: List? = null, override var manual: Any? = null, override val openapi: String? = null, - override val remote: Any? = null + override val remote: Any? = null, + @JsonProperty("change-commit") + @get:Schema(title = "change-commit") + override var changeCommit: Any? = null, + @JsonProperty("change-content") + @get:Schema(title = "change-content") + override var changeContent: Any? = null, + @JsonProperty("change-submit") + @get:Schema(title = "change-submit") + override var changeSubmit: Any? = null, + @JsonProperty("shelve-commit") + @get:Schema(title = "shelve-commit") + override var shelveCommit: Any? = null, + @JsonProperty("shelve-submit") + @get:Schema(title = "shelve-submit") + override var shelveSubmit: Any? = null ) : IPreTriggerOn { override fun yamlVersion() = YamlVersion.V3_0 } diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/on/PushRule.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/on/PushRule.kt index b391f41d848..eff3fd0bdff 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/on/PushRule.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/on/PushRule.kt @@ -40,7 +40,7 @@ import io.swagger.v3.oas.annotations.media.Schema data class PushRule( val name: String? = null, val enable: Boolean? = true, - val branches: List? = listOf("*"), + val branches: List?, @get:Schema(title = "branches-ignore") @JsonProperty("branches-ignore") @@ -62,5 +62,19 @@ data class PushRule( @get:Schema(title = "path-filter-type") @JsonProperty("path-filter-type") - val pathFilterType: String? = null + val pathFilterType: String? = null, + + @get:Schema(title = "custom-filter") + @JsonProperty("custom-filter") + val custom: CustomFilter? = null ) + +data class CustomFilter( + @get:Schema(title = "custom-filter-url") + @JsonProperty("url") + val url: String? = null, + + @get:Schema(title = "custom-filter-credentials") + @JsonProperty("credentials") + val credentials: String? = null +) \ No newline at end of file diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/on/SchedulesRule.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/on/SchedulesRule.kt index d371bbf181d..dcada0d2f90 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/on/SchedulesRule.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/on/SchedulesRule.kt @@ -44,6 +44,9 @@ data class SchedulesRule( val cron: Any? = null, val interval: Interval? = null, + @get:Schema(title = "repo-type") + @JsonProperty("repo-type") + val repoType: String? = null, @get:Schema(title = "repo-id") @JsonProperty("repo-id") val repoId: String? = null, diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/on/TriggerOn.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/on/TriggerOn.kt index 590eaca98e9..df8f444c015 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/on/TriggerOn.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/models/on/TriggerOn.kt @@ -58,7 +58,22 @@ data class TriggerOn( val openapi: String? = null, @JsonProperty("repo-name") @get:Schema(title = "repo-name") - var repoName: String? = null + var repoName: String? = null, + @JsonProperty("change-commit") + @get:Schema(title = "change-commit") + var changeCommit: PushRule? = null, + @JsonProperty("change-content") + @get:Schema(title = "change-content") + var changeContent: PushRule? = null, + @JsonProperty("change-submit") + @get:Schema(title = "change-submit") + var changeSubmit: PushRule? = null, + @JsonProperty("shelve-commit") + @get:Schema(title = "shelve-commit") + var shelveCommit: PushRule? = null, + @JsonProperty("shelve-submit") + @get:Schema(title = "shelve-submit") + var shelveSubmit: PushRule? = null ) { fun toPre(version: YamlVersion) = when (version) { YamlVersion.V2_0 -> toPreV2() @@ -78,7 +93,12 @@ data class TriggerOn( repoHook = null, manual = manual?.let { EnableType.TRUE.value } ?: EnableType.FALSE.value, openapi = openapi, - remote = remote + remote = remote, + changeCommit = changeCommit, + changeContent = changeContent, + changeSubmit = changeSubmit, + shelveCommit = shelveCommit, + shelveSubmit = shelveSubmit ) private fun toPreV3() = PreTriggerOnV3( @@ -96,7 +116,12 @@ data class TriggerOn( repoHook = null, manual = simpleManual(), openapi = openapi, - remote = remote + remote = remote, + changeCommit = changeCommit, + changeContent = changeContent, + changeSubmit = changeSubmit, + shelveCommit = shelveCommit, + shelveSubmit = shelveSubmit ) private fun simpleManual() = when { @@ -120,6 +145,11 @@ interface IPreTriggerOn : YamlVersionParser { val manual: Any? val openapi: String? val remote: Any? + val changeCommit: Any? + val changeSubmit: Any? + val changeContent: Any? + val shelveCommit: Any? + val shelveSubmit: Any? } @JsonInclude(JsonInclude.Include.NON_NULL) @@ -138,7 +168,22 @@ data class PreTriggerOn( override val repoHook: List? = null, override val manual: Any? = null, override val openapi: String? = null, - override val remote: Any? = null + override val remote: Any? = null, + @JsonProperty("change-commit") + @get:Schema(title = "change-commit") + override var changeCommit: Any? = null, + @JsonProperty("change-content") + @get:Schema(title = "change-content") + override var changeContent: Any? = null, + @JsonProperty("change-submit") + @get:Schema(title = "change-submit") + override var changeSubmit: Any? = null, + @JsonProperty("shelve-commit") + @get:Schema(title = "shelve-commit") + override var shelveCommit: Any? = null, + @JsonProperty("shelve-submit") + @get:Schema(title = "shelve-submit") + override var shelveSubmit: Any? = null ) : IPreTriggerOn { override fun yamlVersion() = YamlVersion.V2_0 } diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/parsers/template/YamlObjects.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/parsers/template/YamlObjects.kt index 8a69d35562b..c59d4da5aa4 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/parsers/template/YamlObjects.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/parsers/template/YamlObjects.kt @@ -83,9 +83,12 @@ object YamlObjects { ) // 只有列表需要判断 - if ((va.props?.type == VariablePropType.SELECTOR.value && va.props.datasource == null) || - va.props?.type == VariablePropType.CHECKBOX.value - ) { + if (va.props?.type == VariablePropType.SELECTOR.value || va.props?.type == VariablePropType.CHECKBOX.value) { + // 这期暂不对拉取远程接口的参数做校验 + if (va.props.payload != null) { + return va + } + if (!va.value.isNullOrBlank() && va.props.options.isNullOrEmpty()) { throw YamlFormatException( "$fromPath variable $key format error: value ${va.value} not in variable options" @@ -118,7 +121,6 @@ object YamlObjects { label = getNullValue("label", propsMap), type = getNotNullValue("type", "props", propsMap), options = getVarPropOptions(fromPath, propsMap["options"]), - datasource = getVarPropDataSource(fromPath, propsMap["datasource"]), description = getNullValue("description", propsMap), multiple = getNullValue("multiple", propsMap)?.toBoolean(), required = getNullValue("required", propsMap)?.toBoolean(), @@ -132,11 +134,11 @@ object YamlObjects { key = "metadata", map = propsMap ), - payload = propsMap["glob"] + payload = propsMap["payload"] ) - if (!po.options.isNullOrEmpty() && po.datasource != null) { - throw YamlFormatException("$fromPath variable format error: options and datasource cannot coexist") + if (!po.options.isNullOrEmpty() && po.payload != null) { + throw YamlFormatException("$fromPath variable format error: options and payload cannot coexist") } return po @@ -297,7 +299,8 @@ object YamlObjects { null } else { transValue>(fromPath, "mounts", optionsMap["mounts"]) - } + }, + privileged = getNullValue("privileged", optionsMap)?.toBoolean() ) }, imagePullPolicy = getNullValue(key = "image-pull-policy", map = containerMap) diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/utils/ScriptYmlUtils.kt b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/utils/ScriptYmlUtils.kt index 999ba736b31..3964e10b777 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/utils/ScriptYmlUtils.kt +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/kotlin/com/tencent/devops/process/yaml/v3/utils/ScriptYmlUtils.kt @@ -570,7 +570,7 @@ object ScriptYmlUtils { private fun formatStageLabel(labels: Any?): List { if (labels == null) { - return emptyList() + return listOf(StageLabel.BUILD.id) } val transLabels = anyToListString(labels) @@ -586,6 +586,9 @@ object ScriptYmlUtils { ) } } + if (newLabels.isEmpty()) { + newLabels.add(StageLabel.BUILD.id) + } return newLabels } @@ -631,7 +634,12 @@ object ScriptYmlUtils { note = noteRule(preTriggerOn), manual = manualRule(preTriggerOn), openapi = openapiRule(preTriggerOn), - remote = remoteRule(preTriggerOn) + remote = remoteRule(preTriggerOn), + changeCommit = p4EventRule(preTriggerOn.changeCommit), + changeSubmit = p4EventRule(preTriggerOn.changeSubmit), + changeContent = p4EventRule(preTriggerOn.changeContent), + shelveCommit = p4EventRule(preTriggerOn.shelveCommit), + shelveSubmit = p4EventRule(preTriggerOn.shelveSubmit) ) if (preTriggerOn is PreTriggerOnV3) { @@ -961,6 +969,34 @@ object ScriptYmlUtils { return null } + private fun p4EventRule( + rule: Any? + ): PushRule? { + if (rule != null) { + return try { + YamlUtil.getObjectMapper().readValue( + JsonUtil.toJson(rule), + PushRule::class.java + ) + } catch (e: MismatchedInputException) { + val pushObj = YamlUtil.getObjectMapper().readValue( + JsonUtil.toJson(rule), + List::class.java + ) as ArrayList + + PushRule( + branches = null, + branchesIgnore = null, + paths = pushObj, + pathsIgnore = null, + users = null, + usersIgnore = null + ) + } + } + return null + } + fun validate(schema: String, yamlJson: String): Pair { val schemaNode = jsonNodeFromString(schema) val jsonNode = jsonNodeFromString(yamlJson) diff --git a/src/backend/ci/core/common/common-pipeline-yaml/src/main/resources/schema/V3_0/ci.json b/src/backend/ci/core/common/common-pipeline-yaml/src/main/resources/schema/V3_0/ci.json index c35ac621991..e38f1675aac 100644 --- a/src/backend/ci/core/common/common-pipeline-yaml/src/main/resources/schema/V3_0/ci.json +++ b/src/backend/ci/core/common/common-pipeline-yaml/src/main/resources/schema/V3_0/ci.json @@ -93,6 +93,17 @@ "items" : { "enum" : [ "NamePrefixFilter", "RegexBasedFilter" ] } + }, + "custom-filter" : { + "type" : "object", + "properties" : { + "url" : { + "type" : "string" + }, + "credentials" : { + "type" : "string" + } + } } } } ] @@ -233,6 +244,20 @@ "items" : { "enum" : [ "NamePrefixFilter", "RegexBasedFilter" ] } + }, + "custom-filter" : { + "type" : "object", + "properties" : { + "url" : { + "type" : "string" + }, + "credentials" : { + "type" : "string" + } + } + }, + "skip-wip" : { + "type" : "boolean" } } } ] @@ -272,6 +297,9 @@ } } }, + "repo-type" : { + "enum" : [ "ID", "NAME", "SELF", "NONE" ] + }, "repo-id" : { "type" : "string" }, @@ -325,6 +353,9 @@ } } }, + "repo-type" : { + "enum" : [ "ID", "NAME", "SELF", "NONE" ] + }, "repo-id" : { "type" : "string" }, @@ -524,6 +555,161 @@ "enum" : [ "git", "tgit", "github", "svn", "p4", "gitlab" ] }, "additionalItems" : false + }, + "change-commit" : { + "oneOf" : [ { + "type" : "array", + "items" : { + "type" : "string" + } + }, { + "type" : "object", + "additionalProperties" : false, + "properties" : { + "name" : { + "type" : "string" + }, + "enable" : { + "type" : "boolean" + }, + "paths" : { + "type" : "array", + "items" : { + "type" : "string" + } + }, + "paths-ignore" : { + "type" : "array", + "items" : { + "type" : "string" + } + } + } + } ] + }, + "change-content" : { + "oneOf" : [ { + "type" : "array", + "items" : { + "type" : "string" + } + }, { + "type" : "object", + "additionalProperties" : false, + "properties" : { + "name" : { + "type" : "string" + }, + "enable" : { + "type" : "boolean" + }, + "paths" : { + "type" : "array", + "items" : { + "type" : "string" + } + }, + "paths-ignore" : { + "type" : "array", + "items" : { + "type" : "string" + } + } + } + } ] + }, + "change-submit" : { + "oneOf" : [ { + "type" : "array", + "items" : { + "type" : "string" + } + }, { + "type" : "object", + "additionalProperties" : false, + "properties" : { + "name" : { + "type" : "string" + }, + "enable" : { + "type" : "boolean" + }, + "paths" : { + "type" : "array", + "items" : { + "type" : "string" + } + }, + "paths-ignore" : { + "type" : "array", + "items" : { + "type" : "string" + } + } + } + } ] + }, + "shelve-commit" : { + "oneOf" : [ { + "type" : "array", + "items" : { + "type" : "string" + } + }, { + "type" : "object", + "additionalProperties" : false, + "properties" : { + "name" : { + "type" : "string" + }, + "enable" : { + "type" : "boolean" + }, + "paths" : { + "type" : "array", + "items" : { + "type" : "string" + } + }, + "paths-ignore" : { + "type" : "array", + "items" : { + "type" : "string" + } + } + } + } ] + }, + "shelve-submit" : { + "oneOf" : [ { + "type" : "array", + "items" : { + "type" : "string" + } + }, { + "type" : "object", + "additionalProperties" : false, + "properties" : { + "name" : { + "type" : "string" + }, + "enable" : { + "type" : "boolean" + }, + "paths" : { + "type" : "array", + "items" : { + "type" : "string" + } + }, + "paths-ignore" : { + "type" : "array", + "items" : { + "type" : "string" + } + } + } + } ] } } } @@ -598,6 +784,17 @@ "items" : { "enum" : [ "NamePrefixFilter", "RegexBasedFilter" ] } + }, + "custom-filter" : { + "type" : "object", + "properties" : { + "url" : { + "type" : "string" + }, + "credentials" : { + "type" : "string" + } + } } } } ] @@ -738,6 +935,20 @@ "items" : { "enum" : [ "NamePrefixFilter", "RegexBasedFilter" ] } + }, + "custom-filter" : { + "type" : "object", + "properties" : { + "url" : { + "type" : "string" + }, + "credentials" : { + "type" : "string" + } + } + }, + "skip-wip" : { + "type" : "boolean" } } } ] @@ -777,6 +988,9 @@ } } }, + "repo-type" : { + "enum" : [ "ID", "NAME", "SELF", "NONE" ] + }, "repo-id" : { "type" : "string" }, @@ -830,6 +1044,9 @@ } } }, + "repo-type" : { + "enum" : [ "ID", "NAME", "SELF", "NONE" ] + }, "repo-id" : { "type" : "string" }, @@ -1029,6 +1246,161 @@ "enum" : [ "git", "tgit", "github", "svn", "p4", "gitlab" ] }, "additionalItems" : false + }, + "change-commit" : { + "oneOf" : [ { + "type" : "array", + "items" : { + "type" : "string" + } + }, { + "type" : "object", + "additionalProperties" : false, + "properties" : { + "name" : { + "type" : "string" + }, + "enable" : { + "type" : "boolean" + }, + "paths" : { + "type" : "array", + "items" : { + "type" : "string" + } + }, + "paths-ignore" : { + "type" : "array", + "items" : { + "type" : "string" + } + } + } + } ] + }, + "change-content" : { + "oneOf" : [ { + "type" : "array", + "items" : { + "type" : "string" + } + }, { + "type" : "object", + "additionalProperties" : false, + "properties" : { + "name" : { + "type" : "string" + }, + "enable" : { + "type" : "boolean" + }, + "paths" : { + "type" : "array", + "items" : { + "type" : "string" + } + }, + "paths-ignore" : { + "type" : "array", + "items" : { + "type" : "string" + } + } + } + } ] + }, + "change-submit" : { + "oneOf" : [ { + "type" : "array", + "items" : { + "type" : "string" + } + }, { + "type" : "object", + "additionalProperties" : false, + "properties" : { + "name" : { + "type" : "string" + }, + "enable" : { + "type" : "boolean" + }, + "paths" : { + "type" : "array", + "items" : { + "type" : "string" + } + }, + "paths-ignore" : { + "type" : "array", + "items" : { + "type" : "string" + } + } + } + } ] + }, + "shelve-commit" : { + "oneOf" : [ { + "type" : "array", + "items" : { + "type" : "string" + } + }, { + "type" : "object", + "additionalProperties" : false, + "properties" : { + "name" : { + "type" : "string" + }, + "enable" : { + "type" : "boolean" + }, + "paths" : { + "type" : "array", + "items" : { + "type" : "string" + } + }, + "paths-ignore" : { + "type" : "array", + "items" : { + "type" : "string" + } + } + } + } ] + }, + "shelve-submit" : { + "oneOf" : [ { + "type" : "array", + "items" : { + "type" : "string" + } + }, { + "type" : "object", + "additionalProperties" : false, + "properties" : { + "name" : { + "type" : "string" + }, + "enable" : { + "type" : "boolean" + }, + "paths" : { + "type" : "array", + "items" : { + "type" : "string" + } + }, + "paths-ignore" : { + "type" : "array", + "items" : { + "type" : "string" + } + } + } + } ] } } } ] @@ -1851,8 +2223,8 @@ "description" : { "type" : "string" }, - "send-markdown" : { - "type" : "boolean" + "content-format" : { + "enum" : [ "markdown", "text" ] }, "notify-type" : { "type" : "array", @@ -1860,6 +2232,12 @@ "type" : "string" } }, + "chat-id" : { + "type" : "array", + "items" : { + "type" : "string" + } + }, "notify-groups" : { "type" : "array", "items" : { diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/container/AgentReuseMutex.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/container/AgentReuseMutex.kt index d047f0e4fb9..30facb766aa 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/container/AgentReuseMutex.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/container/AgentReuseMutex.kt @@ -47,8 +47,9 @@ data class AgentReuseMutex( companion object { /** * @see com.tencent.devops.process.engine.common.Timeout MAX_MINUTES + * JOB最长过期时间 7 天对应的秒数 */ - const val AGENT_LOCK_TIMEOUT = 10080L + 2L + const val AGENT_LOCK_TIMEOUT = 60 * 60 * 24 * 7L const val CONTEXT_KEY_SUFFIX = ".container.agent_id" fun genAgentContextKey(jobId: String) = "jobs.$jobId$CONTEXT_KEY_SUFFIX" fun genAgentReuseMutexLockKey(projectId: String, agentId: String) = @@ -58,7 +59,7 @@ data class AgentReuseMutex( "lock:agent:reuse:project:$projectId:agent:$agentId:queue" fun genAgentReuseMutexLinkTipKey(buildId: String): String { - return "linkTip:$buildId" + return "agent:reuse:linkTip:$buildId" } } } diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/container/TriggerContainer.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/container/TriggerContainer.kt index 1a8ea378f51..ecfedd5642b 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/container/TriggerContainer.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/container/TriggerContainer.kt @@ -55,7 +55,7 @@ data class TriggerContainer( @get:Schema(title = "参数化构建", required = false) var params: List = listOf(), @get:Schema(title = "模板参数构建", required = false) - val templateParams: List? = null, + var templateParams: List? = null, @get:Schema(title = "构建版本号", required = false) var buildNo: BuildNo? = null, @get:Schema(title = diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/event/CallBackData.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/event/CallBackData.kt index 540d30ecf09..996d86cb0cb 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/event/CallBackData.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/event/CallBackData.kt @@ -97,6 +97,8 @@ enum class CallBackEvent { BUILD_TASK_END, BUILD_STAGE_START, BUILD_STAGE_END, + BUILD_JOB_START, + BUILD_JOB_END, BUILD_TASK_PAUSE, PROJECT_CREATE, PROJECT_UPDATE, diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/BuildFormProperty.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/BuildFormProperty.kt index 3bf3f022775..a2753a79cc0 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/BuildFormProperty.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/BuildFormProperty.kt @@ -37,7 +37,7 @@ data class BuildFormProperty( var id: String, @get:Schema(title = "元素名称", required = true) var name: String? = null, - @get:Schema(title = "是否必须", required = true) + @get:Schema(title = "是否必须（新前端的入参标识）", required = true) var required: Boolean, @get:Schema(title = "是否为常量", required = true) var constant: Boolean? = false, diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/TemplateInstanceCreateRequest.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/TemplateInstanceCreateRequest.kt index c9febe8f1a4..c9bea63c3e8 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/TemplateInstanceCreateRequest.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/TemplateInstanceCreateRequest.kt @@ -47,8 +47,6 @@ data class TemplateInstanceCreateRequest( var instanceType: String? = PipelineInstanceTypeEnum.FREEDOM.type, @get:Schema(title = "是否为空模板", required = false) var emptyTemplate: Boolean? = false, - @get:Schema(title = "标签", required = false) - var labels: List = emptyList(), @get:Schema(title = "静态流水线组", required = false) var staticViews: List = emptyList() ) diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/Element.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/Element.kt index eff15bcdf2c..c076fbd5176 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/Element.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/Element.kt @@ -149,7 +149,8 @@ abstract class Element( open var progressRate: Double? = null, override var template: String? = null, override var ref: String? = null, - override var variables: Map? = null + override var variables: Map? = null, + var asyncStatus: String? = null ) : IModelTemplate { open fun getAtomCode() = getClassType() diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/agent/LinuxScriptElement.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/agent/LinuxScriptElement.kt index a77dc664a16..3781acf4269 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/agent/LinuxScriptElement.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/agent/LinuxScriptElement.kt @@ -27,6 +27,7 @@ package com.tencent.devops.common.pipeline.pojo.element.agent +import com.tencent.devops.common.pipeline.NameAndValue import com.tencent.devops.common.pipeline.enums.BuildScriptType import com.tencent.devops.common.pipeline.pojo.element.Element import com.tencent.devops.common.pipeline.pojo.element.ElementAdditionalOptions @@ -46,6 +47,8 @@ data class LinuxScriptElement( override var status: String? = null, @get:Schema(title = "用户自定义ID", required = false) override var stepId: String? = null, + @get:Schema(title = "用户自定义环境变量（插件运行时写入环境）", required = false) + override var customEnv: List? = null, @get:Schema(title = "FAQ url链接", required = false) val errorFAQUrl: String? = null, @get:Schema(title = "脚本类型", required = true) diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/agent/WindowsScriptElement.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/agent/WindowsScriptElement.kt index 419bdb7502b..7e9eaa98286 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/agent/WindowsScriptElement.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/agent/WindowsScriptElement.kt @@ -27,6 +27,7 @@ package com.tencent.devops.common.pipeline.pojo.element.agent +import com.tencent.devops.common.pipeline.NameAndValue import com.tencent.devops.common.pipeline.enums.BuildScriptType import com.tencent.devops.common.pipeline.enums.CharsetType import com.tencent.devops.common.pipeline.pojo.element.Element @@ -46,6 +47,8 @@ data class WindowsScriptElement( override var status: String? = null, @get:Schema(title = "用户自定义ID", required = false) override var stepId: String? = null, + @get:Schema(title = "用户自定义环境变量（插件运行时写入环境）", required = false) + override var customEnv: List? = null, @get:Schema(title = "FAQ url链接", required = false) val errorFAQUrl: String? = null, @get:Schema(title = "脚本内容", required = true) diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/atom/ManualReviewParam.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/atom/ManualReviewParam.kt index 1886a231d9f..9f8b8c306a0 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/atom/ManualReviewParam.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/atom/ManualReviewParam.kt @@ -37,7 +37,7 @@ data class ManualReviewParam( @get:Schema(title = "参数名", required = true) var key: String = "", @get:Schema(title = "参数内容(Any 类型)", required = true, type = "string") - var value: Any? = "", + var value: Any? = null, @get:Schema(title = "参数类型", required = false) val valueType: ManualReviewParamType = ManualReviewParamType.STRING, @get:Schema(title = "是否必填", required = true) diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/market/MarketBuildAtomElement.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/market/MarketBuildAtomElement.kt index c47df104b23..81a07554d5e 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/market/MarketBuildAtomElement.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/market/MarketBuildAtomElement.kt @@ -27,6 +27,7 @@ package com.tencent.devops.common.pipeline.pojo.element.market +import com.tencent.devops.common.pipeline.NameAndValue import com.tencent.devops.common.pipeline.pojo.element.Element import com.tencent.devops.common.pipeline.pojo.element.ElementAdditionalOptions import com.tencent.devops.common.pipeline.pojo.transfer.PreStep @@ -48,6 +49,8 @@ data class MarketBuildAtomElement( override var version: String = "1.*", @get:Schema(title = "用户自定义ID", required = false) override var stepId: String? = null, + @get:Schema(title = "用户自定义环境变量（插件运行时写入环境）", required = false) + override var customEnv: List? = null, @get:Schema(title = "插件参数数据", required = true) val data: Map = mapOf(), @get:Schema(title = "附加参数", required = false) diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/market/MarketBuildLessAtomElement.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/market/MarketBuildLessAtomElement.kt index d563e5e71d6..f6cec6d1c97 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/market/MarketBuildLessAtomElement.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/market/MarketBuildLessAtomElement.kt @@ -27,6 +27,7 @@ package com.tencent.devops.common.pipeline.pojo.element.market +import com.tencent.devops.common.pipeline.NameAndValue import com.tencent.devops.common.pipeline.pojo.element.Element import com.tencent.devops.common.pipeline.pojo.transfer.PreStep import com.tencent.devops.common.pipeline.utils.TransferUtil @@ -47,6 +48,8 @@ data class MarketBuildLessAtomElement( override var version: String = "1.*", @get:Schema(title = "用户自定义ID", required = false) override var stepId: String? = null, + @get:Schema(title = "用户自定义环境变量（插件运行时写入环境）", required = false) + override var customEnv: List? = null, @get:Schema(title = "插件参数数据", required = true) val data: Map = mapOf() ) : Element(name, id, status) { diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/matrix/MatrixStatusElement.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/matrix/MatrixStatusElement.kt index 588629abc7d..513a7e738b6 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/matrix/MatrixStatusElement.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/matrix/MatrixStatusElement.kt @@ -27,6 +27,7 @@ package com.tencent.devops.common.pipeline.pojo.element.matrix +import com.tencent.devops.common.pipeline.NameAndValue import com.tencent.devops.common.pipeline.pojo.element.Element import io.swagger.v3.oas.annotations.media.Schema @@ -46,6 +47,8 @@ data class MatrixStatusElement( override var startEpoch: Long? = null, @get:Schema(title = "上下文标识", required = false) override var stepId: String?, + @get:Schema(title = "用户自定义环境变量（插件运行时写入环境）", required = false) + override var customEnv: List? = null, @get:Schema(title = "原插件的类型标识") val originClassType: String, @get:Schema(title = "原插件的市场标识") diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/trigger/CodeGitlabWebHookTriggerElement.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/trigger/CodeGitlabWebHookTriggerElement.kt index 2bded0086f2..2e036ac8943 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/trigger/CodeGitlabWebHookTriggerElement.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/trigger/CodeGitlabWebHookTriggerElement.kt @@ -79,7 +79,11 @@ data class CodeGitlabWebHookTriggerElement( @get:Schema(title = "用于包含的提交信息", required = false) val includeCommitMsg: String? = null, @get:Schema(title = "用于排除的提交信息", required = false) - val excludeCommitMsg: String? = null + val excludeCommitMsg: String? = null, + @get:Schema(title = "push事件action") + val includePushAction: List? = null, + @get:Schema(title = "mr事件action") + val includeMrAction: List? = null ) : WebHookTriggerElement(name, id, status) { companion object { const val classType = "codeGitlabWebHookTrigger" @@ -100,6 +104,7 @@ data class CodeGitlabWebHookTriggerElement( val props = when (eventType) { CodeEventType.PUSH -> { listOf( + vuexInput(name = "action", value = joinToString(includePushAction)), vuexInput(name = "branchName", value = branchName), vuexInput(name = "excludeBranchName", value = excludeBranchName), vuexInput(name = "includePaths", value = includePaths), @@ -111,6 +116,7 @@ data class CodeGitlabWebHookTriggerElement( CodeEventType.MERGE_REQUEST -> { listOf( + vuexInput(name = "action", value = joinToString(includeMrAction)), vuexInput(name = "branchName", value = branchName), vuexInput(name = "excludeBranchName", value = excludeBranchName), vuexInput( diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/trigger/TimerTriggerElement.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/trigger/TimerTriggerElement.kt index c36ed6b7322..5fdfaff30e7 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/trigger/TimerTriggerElement.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/element/trigger/TimerTriggerElement.kt @@ -31,7 +31,7 @@ import com.cronutils.mapper.CronMapper import com.cronutils.model.CronType import com.cronutils.model.definition.CronDefinitionBuilder import com.cronutils.parser.CronParser -import com.tencent.devops.common.api.enums.RepositoryType +import com.tencent.devops.common.api.enums.TriggerRepositoryType import com.tencent.devops.common.api.util.EnvUtils import com.tencent.devops.common.pipeline.enums.StartType import com.tencent.devops.common.pipeline.pojo.element.Element @@ -58,7 +58,7 @@ data class TimerTriggerElement( @get:Schema(title = "指定代码库分支", required = false) val branches: List? = null, @get:Schema(title = "代码库类型", required = false) - val repositoryType: RepositoryType? = null, + val repositoryType: TriggerRepositoryType? = null, @get:Schema(title = "代码库HashId", required = false) val repoHashId: String? = null, @get:Schema(title = "指定代码库别名", required = false) @@ -134,4 +134,10 @@ data class TimerTriggerElement( super.findFirstTaskIdByStartType(startType) } } + + fun enableRepoConfig(): Boolean { + return repositoryType == TriggerRepositoryType.SELF || + repositoryType == TriggerRepositoryType.ID && !repoHashId.isNullOrBlank() || + repositoryType == TriggerRepositoryType.NAME && !repoName.isNullOrBlank() + } } diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/setting/PipelineRunLockType.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/setting/PipelineRunLockType.kt index 1b2d2f97a66..9ae84dc8761 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/setting/PipelineRunLockType.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/setting/PipelineRunLockType.kt @@ -43,6 +43,9 @@ enum class PipelineRunLockType { GROUP_LOCK; companion object { + /** + * 注意，数字与枚举的ordinal不一样，ordinal是下标为0开始 ，而这以1为开始 + */ fun toValue(type: PipelineRunLockType?): Int { return when (type) { null -> 1 @@ -64,11 +67,5 @@ enum class PipelineRunLockType { else -> MULTIPLE } } - - fun checkLock(runLockType: Int?): Boolean { - return if (runLockType == null) { - false - } else valueOf(runLockType) == LOCK - } } } diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/setting/PipelineSetting.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/setting/PipelineSetting.kt index 13729d2373d..087448cd44d 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/setting/PipelineSetting.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/setting/PipelineSetting.kt @@ -63,10 +63,10 @@ data class PipelineSetting( // 通知订阅相关配置 @Deprecated("被successSubscriptionList取代") @get:Schema(title = "订阅成功相关", required = false) - var successSubscription: Subscription = Subscription(), + var successSubscription: Subscription? = Subscription(), @Deprecated("被failSubscriptionList取代") @get:Schema(title = "订阅失败相关", required = false) - var failSubscription: Subscription = Subscription(), + var failSubscription: Subscription? = Subscription(), @get:Schema(title = "订阅成功通知组", required = false) var successSubscriptionList: List? = null, @get:Schema(title = "订阅失败通知组", required = false) @@ -117,7 +117,9 @@ data class PipelineSetting( ), maxQueueSize = PIPELINE_SETTING_MAX_QUEUE_SIZE_DEFAULT, runLockType = PipelineRunLockType.MULTIPLE, - failSubscription = failSubscription ?: Subscription(), + successSubscription = null, + failSubscription = null, + successSubscriptionList = emptyList(), failSubscriptionList = failSubscription?.let { listOf(it) }, pipelineAsCodeSettings = PipelineAsCodeSettings() ) @@ -137,10 +139,10 @@ data class PipelineSetting( ) { res = false } - if (this.successSubscription.types.isNotEmpty()) { + if (this.successSubscription?.types?.isNotEmpty() == true) { res = false } - if (this.failSubscription.types.isNotEmpty()) { + if (this.failSubscription?.types?.isNotEmpty() == true) { res = false } return res @@ -153,13 +155,13 @@ data class PipelineSetting( fun fixSubscriptions() { // 只有旧数据向新数据的更新，取消旧数据的保存 - if (successSubscriptionList.isNullOrEmpty()) { - successSubscriptionList = listOf(this.successSubscription) + if (successSubscriptionList == null && this.successSubscription != null) { + successSubscriptionList = listOf(this.successSubscription!!) } - successSubscription = successSubscriptionList!!.first() - if (failSubscriptionList.isNullOrEmpty()) { - failSubscriptionList = listOf(this.failSubscription) + successSubscription = successSubscriptionList!!.firstOrNull() + if (failSubscriptionList == null && this.failSubscription != null) { + failSubscriptionList = listOf(this.failSubscription!!) } - failSubscription = failSubscriptionList!!.first() + failSubscription = failSubscriptionList!!.firstOrNull() } } diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/setting/PipelineSubscriptionType.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/setting/PipelineSubscriptionType.kt index 999a05752bd..13d2add607a 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/setting/PipelineSubscriptionType.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/setting/PipelineSubscriptionType.kt @@ -36,5 +36,7 @@ enum class PipelineSubscriptionType { WECHAT, SMS, WEWORK, - VOICE + VOICE, + // 企业微信群通知 + WEWORK_GROUP } diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/setting/Subscription.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/setting/Subscription.kt index e22799a50a4..e5e7a5655c0 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/setting/Subscription.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/pojo/setting/Subscription.kt @@ -47,4 +47,18 @@ data class Subscription( val detailFlag: Boolean = false, @get:Schema(title = "自定义通知内容", required = false) val content: String = "" -) +) { + + // 转换企业微信组通知 + fun fixWeworkGroupType(): Subscription { + val fixTypes = if (wechatGroupFlag && !types.contains(PipelineSubscriptionType.WEWORK_GROUP)) { + val fixTypes = mutableSetOf() + fixTypes.addAll(types) + fixTypes.add(PipelineSubscriptionType.WEWORK_GROUP) + fixTypes + } else { + types + } + return this.copy(types = fixTypes, wechatGroupFlag = false) + } +} diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/agent/ThirdPartyAgentDispatch.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/agent/ThirdPartyAgentDispatch.kt index 2251e129d8f..ecf25829ff2 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/agent/ThirdPartyAgentDispatch.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/agent/ThirdPartyAgentDispatch.kt @@ -12,7 +12,10 @@ abstract class ThirdPartyAgentDispatch( // 类型为REUSE_JOB时，被复用的job的value，防止同一个stage并发下拿不到agent，启动时填充 open var reusedInfo: ReusedInfo? ) : DispatchType(value) { - fun idType(): Boolean = (agentType == AgentType.ID) || (reusedInfo?.agentType == AgentType.ID) + // 本身是 id，和被复用对象在同一JOB且被复用对象也是 id，是复用但是位于后面的 JOB + fun idType(): Boolean = + (agentType == AgentType.ID) || (reusedInfo?.agentType == AgentType.ID) || + (agentType == AgentType.REUSE_JOB_ID && reusedInfo == null) // 是否在复用锁定链上 fun hasReuseMutex(): Boolean = this.agentType.isReuse() || this.reusedInfo != null diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/agent/ThirdPartyAgentDockerInfo.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/agent/ThirdPartyAgentDockerInfo.kt index b04bcbdd286..82802acc32b 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/agent/ThirdPartyAgentDockerInfo.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/type/agent/ThirdPartyAgentDockerInfo.kt @@ -32,6 +32,11 @@ fun ThirdPartyAgentDockerInfo.replaceField(variables: Map) { } else { EnvUtils.parseEnv(options?.gpus, variables) } + options?.privileged = if (options?.privileged == null) { + null + } else { + EnvUtils.parseEnv(options?.privileged.toString(), variables).toBoolean() + } } if (!imagePullPolicy.isNullOrBlank()) { imagePullPolicy = EnvUtils.parseEnv(imagePullPolicy, variables) @@ -54,7 +59,8 @@ data class Credential( data class DockerOptions( var volumes: List?, var mounts: List?, - var gpus: String? + var gpus: String?, + var privileged: Boolean? ) // 第三方构建机docker类型，调度使用，会带有调度相关信息 diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/utils/EventUtils.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/utils/EventUtils.kt new file mode 100644 index 00000000000..e2ba28f9420 --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/utils/EventUtils.kt @@ -0,0 +1,77 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.common.pipeline.utils + +import com.tencent.devops.common.event.enums.ActionType +import com.tencent.devops.common.event.pojo.pipeline.PipelineBuildStatusBroadCastEvent +import com.tencent.devops.common.pipeline.event.CallBackEvent + +@Suppress("ComplexMethod") +object EventUtils { + fun PipelineBuildStatusBroadCastEvent.toEventType(): CallBackEvent? { + if (!taskId.isNullOrBlank()) { + if (actionType == ActionType.START) { + return CallBackEvent.BUILD_TASK_START + } + if (actionType == ActionType.REFRESH) { + return CallBackEvent.BUILD_TASK_PAUSE + } + if (actionType == ActionType.END) { + return CallBackEvent.BUILD_TASK_END + } + } + + if (!containerHashId.isNullOrBlank()) { + if (actionType == ActionType.START) { + return CallBackEvent.BUILD_JOB_START + } + if (actionType == ActionType.END) { + return CallBackEvent.BUILD_JOB_END + } + } + + if (!stageId.isNullOrBlank()) { + if (actionType == ActionType.START) { + return CallBackEvent.BUILD_STAGE_START + } + if (actionType == ActionType.END) { + return CallBackEvent.BUILD_STAGE_END + } + } + + if (taskId.isNullOrBlank() && containerHashId.isNullOrBlank() && stageId.isNullOrBlank()) { + if (actionType == ActionType.START) { + return CallBackEvent.BUILD_START + } + if (actionType == ActionType.END) { + return CallBackEvent.BUILD_END + } + } + return null + } +} diff --git a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/utils/MatrixContextUtils.kt b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/utils/MatrixContextUtils.kt index 81abbf7385a..8470a397784 100644 --- a/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/utils/MatrixContextUtils.kt +++ b/src/backend/ci/core/common/common-pipeline/src/main/kotlin/com/tencent/devops/common/pipeline/utils/MatrixContextUtils.kt @@ -124,7 +124,9 @@ object MatrixContextUtils { } """ - private val yaml = Yaml() + private val yaml = ThreadLocal.withInitial { + Yaml() + } private val schemaFactory = JsonSchemaFactory.builder(JsonSchemaFactory.getInstance(SpecVersion.VersionFlag.V7)) .objectMapper(YamlUtil.getObjectMapper()) @@ -151,7 +153,7 @@ object MatrixContextUtils { if (originYaml.isBlank()) { return } - val yamlJson = YamlUtil.getObjectMapper().readTree(YamlUtil.toYaml(yaml.load(originYaml))).replaceOn() + val yamlJson = YamlUtil.getObjectMapper().readTree(YamlUtil.toYaml(yaml.get().load(originYaml))).replaceOn() schemaFactory.getSchema(schemaJson).check(yamlJson) } diff --git a/src/backend/ci/core/common/common-pipeline/src/test/kotlin/com/tencent/devops/common/pipeline/pojo/element/ElementTest.kt b/src/backend/ci/core/common/common-pipeline/src/test/kotlin/com/tencent/devops/common/pipeline/pojo/element/ElementTest.kt index 92ec65298b1..a7b40dd8c11 100644 --- a/src/backend/ci/core/common/common-pipeline/src/test/kotlin/com/tencent/devops/common/pipeline/pojo/element/ElementTest.kt +++ b/src/backend/ci/core/common/common-pipeline/src/test/kotlin/com/tencent/devops/common/pipeline/pojo/element/ElementTest.kt @@ -28,8 +28,10 @@ package com.tencent.devops.common.pipeline.pojo.element import com.tencent.devops.common.api.util.JsonUtil +import com.tencent.devops.common.pipeline.enums.BuildScriptType import com.tencent.devops.common.pipeline.enums.BuildStatus import com.tencent.devops.common.pipeline.enums.StartType +import com.tencent.devops.common.pipeline.pojo.element.agent.WindowsScriptElement import com.tencent.devops.common.pipeline.pojo.element.market.MarketBuildAtomElement import com.tencent.devops.common.pipeline.pojo.element.market.MarketBuildLessAtomElement import com.tencent.devops.common.pipeline.pojo.element.trigger.CodeGitWebHookTriggerElement @@ -46,6 +48,21 @@ import org.junit.jupiter.api.Test class ElementTest { + @Test + fun testElementJsonOrder() { + val jsonFile = ElementTest::class.java.classLoader.getResource("windowsElement.json") + val expected = jsonFile!!.readText().trim('\n') + val wel = WindowsScriptElement( + id = "e-326ce1c320204980a3d2a0f241bccd63", + name = "batch script", + script = "unity build", + scriptType = BuildScriptType.BAT + ) + wel.additionalOptions = elementAdditionalOptions() + val actual = JsonUtil.toSortJson(wel) + assertEquals(expected, actual) + } + @Test fun unknownSubType() { val json = """{ diff --git a/src/backend/ci/core/common/common-pipeline/src/test/resources/windowsElement.json b/src/backend/ci/core/common/common-pipeline/src/test/resources/windowsElement.json new file mode 100644 index 00000000000..db939260845 --- /dev/null +++ b/src/backend/ci/core/common/common-pipeline/src/test/resources/windowsElement.json @@ -0,0 +1,25 @@ +{ + "@type" : "windowsScript", + "additionalOptions" : { + "continueWhenFailed" : false, + "enable" : true, + "enableCustomEnv" : true, + "manualRetry" : true, + "pauseBeforeExec" : false, + "retryCount" : 0, + "retryWhenFailed" : false, + "runCondition" : "PRE_TASK_SUCCESS", + "subscriptionPauseUser" : "", + "timeout" : 100 + }, + "atomCode" : "windowsScript", + "classType" : "windowsScript", + "elementEnable" : true, + "executeCount" : 1, + "id" : "e-326ce1c320204980a3d2a0f241bccd63", + "name" : "batch script", + "script" : "unity build", + "scriptType" : "BAT", + "taskAtom" : "", + "version" : "1.*" +} diff --git a/src/backend/ci/core/common/common-quality/src/main/kotlin/com/tencent/devops/common/quality/pojo/QualityRuleInterceptRecord.kt b/src/backend/ci/core/common/common-quality/src/main/kotlin/com/tencent/devops/common/quality/pojo/QualityRuleInterceptRecord.kt index 70f07c78b5c..d6cad4585a1 100644 --- a/src/backend/ci/core/common/common-quality/src/main/kotlin/com/tencent/devops/common/quality/pojo/QualityRuleInterceptRecord.kt +++ b/src/backend/ci/core/common/common-quality/src/main/kotlin/com/tencent/devops/common/quality/pojo/QualityRuleInterceptRecord.kt @@ -51,5 +51,7 @@ data class QualityRuleInterceptRecord( @get:Schema(title = "指标详情", required = true) val detail: String?, @get:Schema(title = "指标日志输出详情", required = false) - var logPrompt: String? + var logPrompt: String?, + @get:Schema(title = "控制点的插件id", required = false) + val controlPointElementId: String? = "" ) diff --git a/src/backend/ci/core/common/common-redis/src/main/kotlin/com/tencent/devops/common/redis/RedisOperation.kt b/src/backend/ci/core/common/common-redis/src/main/kotlin/com/tencent/devops/common/redis/RedisOperation.kt index d1d9195ecb5..1f830a06cfd 100644 --- a/src/backend/ci/core/common/common-redis/src/main/kotlin/com/tencent/devops/common/redis/RedisOperation.kt +++ b/src/backend/ci/core/common/common-redis/src/main/kotlin/com/tencent/devops/common/redis/RedisOperation.kt @@ -29,15 +29,15 @@ package com.tencent.devops.common.redis import com.tencent.devops.common.redis.split.RedisSplitProperties import io.micrometer.core.instrument.util.NamedThreadFactory +import java.util.Date +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit import org.springframework.data.redis.core.Cursor import org.springframework.data.redis.core.DefaultTypedTuple import org.springframework.data.redis.core.RedisCallback import org.springframework.data.redis.core.RedisTemplate import org.springframework.data.redis.core.ScanOptions import org.springframework.data.redis.core.script.RedisScript -import java.util.Date -import java.util.concurrent.Executors -import java.util.concurrent.TimeUnit @Suppress("TooManyFunctions", "UNUSED", "ComplexMethod") class RedisOperation( @@ -228,6 +228,11 @@ class RedisOperation( return masterRedisTemplate.opsForHash().get(getFinalKey(key, isDistinguishCluster), hashKey) } + fun hmGet(key: String, hashKeys: Collection, isDistinguishCluster: Boolean? = false): List? { + return masterRedisTemplate.opsForHash() + .multiGet(getFinalKey(key, isDistinguishCluster), hashKeys) + } + fun hdelete(key: String, hashKey: String, isDistinguishCluster: Boolean? = false) { // 双写 writeSlaveIfNeed { @@ -236,12 +241,12 @@ class RedisOperation( masterRedisTemplate.opsForHash().delete(getFinalKey(key, isDistinguishCluster), hashKey) } - fun hdelete(key: String, hashKeys: Collection, isDistinguishCluster: Boolean? = false) { + fun hdelete(key: String, hashKeys: Array, isDistinguishCluster: Boolean? = false) { // 双写 writeSlaveIfNeed { - slaveRedisTemplate!!.opsForHash().delete(getFinalKey(key, isDistinguishCluster), hashKeys) + slaveRedisTemplate!!.opsForHash().delete(getFinalKey(key, isDistinguishCluster), *hashKeys) } - masterRedisTemplate.opsForHash().delete(getFinalKey(key, isDistinguishCluster), hashKeys) + masterRedisTemplate.opsForHash().delete(getFinalKey(key, isDistinguishCluster), *hashKeys) } fun hhaskey(key: String, hashKey: String, isDistinguishCluster: Boolean? = false): Boolean { diff --git a/src/backend/ci/core/common/common-service/src/main/kotlin/com/tencent/devops/common/service/utils/CommonUtils.kt b/src/backend/ci/core/common/common-service/src/main/kotlin/com/tencent/devops/common/service/utils/CommonUtils.kt index b2c6d7aaaa9..082e862d9ac 100755 --- a/src/backend/ci/core/common/common-service/src/main/kotlin/com/tencent/devops/common/service/utils/CommonUtils.kt +++ b/src/backend/ci/core/common/common-service/src/main/kotlin/com/tencent/devops/common/service/utils/CommonUtils.kt @@ -34,11 +34,9 @@ import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.api.util.JsonUtil import com.tencent.devops.common.api.util.MessageUtil import com.tencent.devops.common.api.util.OkhttpUtils -import com.tencent.devops.common.service.PROFILE_AUTO import com.tencent.devops.common.service.PROFILE_DEFAULT import com.tencent.devops.common.service.PROFILE_DEVELOPMENT import com.tencent.devops.common.service.PROFILE_PRODUCTION -import com.tencent.devops.common.service.PROFILE_STREAM import com.tencent.devops.common.service.PROFILE_TEST import com.tencent.devops.common.service.Profile import org.apache.commons.lang3.StringUtils @@ -235,17 +233,7 @@ object CommonUtils { PROFILE_TEST } profile.isProd() -> { - when { - profile.isAuto() -> { - PROFILE_AUTO - } - profile.isStream() -> { - PROFILE_STREAM - } - else -> { - PROFILE_PRODUCTION - } - } + getProdDbClusterName(profile) } profile.isLocal() -> { PROFILE_DEFAULT @@ -256,6 +244,36 @@ object CommonUtils { } } + private fun getProdDbClusterName(profile: Profile): String { + // 从配置文件获取db集群名称列表 + val dbClusterNames = (SpringContextUtil.getValue("bk.db.clusterNames") ?: PROFILE_PRODUCTION).split(",") + val activeProfiles = profile.getActiveProfiles() + var finalDbClusterName = PROFILE_PRODUCTION + run breaking@{ + // 获取当前服务器集群对应的db集群名称 + activeProfiles.forEach { activeProfile -> + val dbClusterName = getDbClusterNameByProfile(dbClusterNames, activeProfile) + dbClusterName?.let { + finalDbClusterName = dbClusterName + return@breaking + } + } + } + return finalDbClusterName + } + + private fun getDbClusterNameByProfile( + dbClusterNames: List, + activeProfile: String + ): String? { + dbClusterNames.forEach { dbClusterName -> + if (activeProfile.contains(dbClusterName)) { + return dbClusterName + } + } + return null + } + /** * 获取jooq上下文对象 * @param archiveFlag 归档标识 diff --git a/src/backend/ci/core/common/common-service/src/main/kotlin/com/tencent/devops/common/service/utils/LogUtils.kt b/src/backend/ci/core/common/common-service/src/main/kotlin/com/tencent/devops/common/service/utils/LogUtils.kt index eacf41db155..094f0e1544b 100644 --- a/src/backend/ci/core/common/common-service/src/main/kotlin/com/tencent/devops/common/service/utils/LogUtils.kt +++ b/src/backend/ci/core/common/common-service/src/main/kotlin/com/tencent/devops/common/service/utils/LogUtils.kt @@ -64,4 +64,33 @@ object LogUtils { } } } + + /** + * 获取有限长度的日志内容，默认最大长度为16K + * @param logStr 原始日志内容 + * @return 截取后的日志 + */ + fun getLogWithLengthLimit(logStr: String?): String? { + val defaultMaxLength = 16384 + return getLogWithLengthLimit(logStr, defaultMaxLength) + } + + /** + * 获取有限长度的日志内容 + * @param logStr 原始日志内容 + * @param maxLength 最大长度，若小于0则不生效，返回原始日志 + * @return 截取后的日志 + */ + fun getLogWithLengthLimit(logStr: String?, maxLength: Int): String? { + if (logStr == null) { + return null + } + if (maxLength < 0) { + return logStr + } + return if (logStr.length > maxLength) + logStr.substring(0, maxLength) + else + logStr + } } diff --git a/src/backend/ci/core/common/common-service/src/main/kotlin/com/tencent/devops/common/service/utils/SpringContextUtil.kt b/src/backend/ci/core/common/common-service/src/main/kotlin/com/tencent/devops/common/service/utils/SpringContextUtil.kt index a7f93d85b25..52d8924d1b4 100644 --- a/src/backend/ci/core/common/common-service/src/main/kotlin/com/tencent/devops/common/service/utils/SpringContextUtil.kt +++ b/src/backend/ci/core/common/common-service/src/main/kotlin/com/tencent/devops/common/service/utils/SpringContextUtil.kt @@ -105,5 +105,14 @@ class SpringContextUtil : ApplicationContextAware { fun getValue(key: String): String? { return applicationContext?.environment?.get(key) } + + /** + * 根据bean名称判断bean是否存在 + * @param beanName bean名称 + * @return 布尔值 + */ + fun isBeanExist(beanName: String): Boolean { + return applicationContext?.containsBean(beanName) ?: false + } } } diff --git a/src/backend/ci/core/common/common-web/src/main/kotlin/com/tencent/devops/common/web/aop/SensitiveApiPermissionAspect.kt b/src/backend/ci/core/common/common-web/src/main/kotlin/com/tencent/devops/common/web/aop/SensitiveApiPermissionAspect.kt index c257609b90f..03509df1567 100644 --- a/src/backend/ci/core/common/common-web/src/main/kotlin/com/tencent/devops/common/web/aop/SensitiveApiPermissionAspect.kt +++ b/src/backend/ci/core/common/common-web/src/main/kotlin/com/tencent/devops/common/web/aop/SensitiveApiPermissionAspect.kt @@ -29,7 +29,14 @@ package com.tencent.devops.common.web.aop import com.github.benmanes.caffeine.cache.Caffeine import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_BUILD_ID +import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_OS_ARCH +import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_OS_NAME +import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_SHA_CONTENT +import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_STORE_CODE +import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_STORE_TYPE +import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_STORE_VERSION import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_VM_SEQ_ID +import com.tencent.devops.common.api.auth.REFERER import com.tencent.devops.common.api.auth.SIGN_HEADER_NONCE import com.tencent.devops.common.api.auth.SIGN_HEADER_TIMESTAMP import com.tencent.devops.common.api.auth.SING_HEADER_SIGNATURE @@ -37,6 +44,7 @@ import com.tencent.devops.common.api.constant.CommonMessageCode import com.tencent.devops.common.api.exception.ErrorCodeException import com.tencent.devops.common.client.Client import com.tencent.devops.common.redis.RedisOperation +import com.tencent.devops.common.service.PROFILE_DEVX import com.tencent.devops.common.util.ApiSignUtil import com.tencent.devops.common.web.annotation.SensitiveApiPermission import com.tencent.devops.common.web.service.ServiceSensitiveApiPermissionResource @@ -63,53 +71,91 @@ class SensitiveApiPermissionAspect constructor( private val apiPermissionCache = Caffeine.newBuilder() .maximumSize(CACHE_MAX_SIZE) - .build() + .build() @Before("pointCut()") fun doBefore(jp: JoinPoint) { val request = (RequestContextHolder.getRequestAttributes() as ServletRequestAttributes).request - val buildId = request.getHeader(AUTH_HEADER_DEVOPS_BUILD_ID) - val vmSeqId = request.getHeader(AUTH_HEADER_DEVOPS_VM_SEQ_ID) val method = (jp.signature as MethodSignature).method val apiName = method.getAnnotation(SensitiveApiPermission::class.java)?.value - - val (atomCode, signToken) = if (buildId != null && vmSeqId != null) { - AtomRuntimeUtil.getRunningAtomValue( - redisOperation = redisOperation, buildId = buildId, vmSeqId = vmSeqId - ) ?: Pair(null, null) + val referer = request.getHeader(REFERER) + var verifyFlag = false + var storeCode: String? = request.getHeader(AUTH_HEADER_DEVOPS_STORE_CODE) + if (referer?.contains(PROFILE_DEVX) == true) { + verifyFlag = doShaValidateBus(request, storeCode, apiName) } else { - Pair(null, null) + val buildId = request.getHeader(AUTH_HEADER_DEVOPS_BUILD_ID) + val vmSeqId = request.getHeader(AUTH_HEADER_DEVOPS_VM_SEQ_ID) + val (atomCode, signToken) = if (buildId != null && vmSeqId != null) { + AtomRuntimeUtil.getRunningAtomValue( + redisOperation = redisOperation, buildId = buildId, vmSeqId = vmSeqId + ) ?: Pair(null, null) + } else { + Pair(null, null) + } + storeCode = atomCode + if (apiName != null && storeCode != null) { + verifyFlag = verifyToken(request, signToken) || verifyApi("ATOM", storeCode, apiName) + } } - if (apiName != null && atomCode != null) { - logger.info("$buildId|$vmSeqId|$atomCode|$apiName|using sensitive api") - if (enableSensitiveApi && - !verifyToken(request, signToken) && - !verifyApi(buildId, vmSeqId, atomCode, apiName) - ) { - logger.warn("$buildId|$vmSeqId|$atomCode|$apiName|verify sensitive api failed") - throw ErrorCodeException( - statusCode = 401, - errorCode = CommonMessageCode.ERROR_SENSITIVE_API_NO_AUTH, - params = arrayOf(atomCode, apiName), - defaultMessage = "Unauthorized: sensitive api $apiName cannot be used by $atomCode" - ) - } + logger.info("$storeCode|$apiName|using sensitive api") + if (enableSensitiveApi && !verifyFlag) { + logger.warn("$storeCode|$apiName|verify sensitive api failed") + throw ErrorCodeException( + statusCode = 401, + errorCode = CommonMessageCode.ERROR_SENSITIVE_API_NO_AUTH, + params = arrayOf(storeCode ?: "", apiName ?: ""), + defaultMessage = "Unauthorized: sensitive api $apiName cannot be used by $storeCode" + ) + } + } + + private fun doShaValidateBus( + request: HttpServletRequest, + storeCode: String?, + apiName: String? + ): Boolean { + val storeType = request.getHeader(AUTH_HEADER_DEVOPS_STORE_TYPE) + val version = request.getHeader(AUTH_HEADER_DEVOPS_STORE_VERSION) + val checkParamFlag = !storeType.isNullOrBlank() && !version.isNullOrBlank() + if (checkParamFlag && !apiName.isNullOrBlank() && !storeCode.isNullOrBlank()) { + val installedPkgShaContent = request.getHeader(AUTH_HEADER_DEVOPS_SHA_CONTENT) + val osName = request.getHeader(AUTH_HEADER_DEVOPS_OS_NAME) + val osArch = request.getHeader(AUTH_HEADER_DEVOPS_OS_ARCH) + return verifyApi( + storeType = storeType, + storeCode = storeCode, + apiName = apiName, + version = version, + installedPkgShaContent = installedPkgShaContent, + osName = osName, + osArch = osArch + ) } + return false } + @Suppress("LongParameterList") private fun verifyApi( - buildId: String, - vmSeqId: String, - atomCode: String, - apiName: String + storeType: String, + storeCode: String, + apiName: String, + version: String? = null, + installedPkgShaContent: String? = null, + osName: String? = null, + osArch: String? = null ): Boolean { - val cacheKey = "$atomCode:$apiName" - logger.info("buildId:$buildId|vmSeqId:$vmSeqId|atomCode:$atomCode|apiName:$apiName|verify sensitive api") + val cacheKey = "$storeType:$storeCode:$apiName" return apiPermissionCache.getIfPresent(cacheKey) ?: run { val apiPermission = client.get(ServiceSensitiveApiPermissionResource::class).verifyApi( - atomCode = atomCode, - apiName = apiName + installedPkgShaContent = installedPkgShaContent, + osName = osName, + osArch = osArch, + storeCode = storeCode, + apiName = apiName, + storeType = storeType, + version = version ).data == true // 只有验证通过的插件才缓存,没有验证通过的插件状态是动态的 if (apiPermission) { diff --git a/src/backend/ci/core/common/common-web/src/main/kotlin/com/tencent/devops/common/web/constant/BkApiHandleType.kt b/src/backend/ci/core/common/common-web/src/main/kotlin/com/tencent/devops/common/web/constant/BkApiHandleType.kt index 5748f766e9b..176e5ad332b 100644 --- a/src/backend/ci/core/common/common-web/src/main/kotlin/com/tencent/devops/common/web/constant/BkApiHandleType.kt +++ b/src/backend/ci/core/common/common-web/src/main/kotlin/com/tencent/devops/common/web/constant/BkApiHandleType.kt @@ -31,5 +31,6 @@ enum class BkApiHandleType { BUILD_API_AUTH_CHECK, // build接口权限校验 PROJECT_API_ACCESS_LIMIT, // 限制项目接口访问权限 PIPELINE_API_ACCESS_LIMIT, // 限制流水线接口访问权限 - API_NO_AUTH_CHECK // 接口免权限校验 + API_NO_AUTH_CHECK, // 接口免权限校验 + API_OPEN_TOKEN_CHECK // open接口token校验 } diff --git a/src/backend/ci/core/common/common-web/src/main/kotlin/com/tencent/devops/common/web/factory/BkApiHandleFactory.kt b/src/backend/ci/core/common/common-web/src/main/kotlin/com/tencent/devops/common/web/factory/BkApiHandleFactory.kt index edcdfdbd59e..8bbdf2179fe 100644 --- a/src/backend/ci/core/common/common-web/src/main/kotlin/com/tencent/devops/common/web/factory/BkApiHandleFactory.kt +++ b/src/backend/ci/core/common/common-web/src/main/kotlin/com/tencent/devops/common/web/factory/BkApiHandleFactory.kt @@ -30,6 +30,7 @@ package com.tencent.devops.common.web.factory import com.tencent.devops.common.web.constant.BkApiHandleType import com.tencent.devops.common.web.service.BkApiHandleService import com.tencent.devops.common.web.service.impl.BkApiHandleBuildAuthServiceImpl +import com.tencent.devops.common.web.service.impl.BkApiHandleOpenAccessServiceImpl import com.tencent.devops.common.web.service.impl.BkApiHandlePipelineAccessServiceImpl import com.tencent.devops.common.web.service.impl.BkApiHandleProjectAccessServiceImpl import java.util.concurrent.ConcurrentHashMap @@ -64,6 +65,13 @@ object BkApiHandleFactory { } } + BkApiHandleType.API_OPEN_TOKEN_CHECK -> { + if (bkApiHandleService == null) { + bkApiHandleService = BkApiHandleOpenAccessServiceImpl() + bkApiHandleMap[type.name] = bkApiHandleService + } + } + else -> {} } return bkApiHandleService diff --git a/src/backend/ci/core/common/common-web/src/main/kotlin/com/tencent/devops/common/web/service/ServiceSensitiveApiPermissionResource.kt b/src/backend/ci/core/common/common-web/src/main/kotlin/com/tencent/devops/common/web/service/ServiceSensitiveApiPermissionResource.kt index dae35d048c6..037cb53203b 100644 --- a/src/backend/ci/core/common/common-web/src/main/kotlin/com/tencent/devops/common/web/service/ServiceSensitiveApiPermissionResource.kt +++ b/src/backend/ci/core/common/common-web/src/main/kotlin/com/tencent/devops/common/web/service/ServiceSensitiveApiPermissionResource.kt @@ -28,16 +28,22 @@ package com.tencent.devops.common.web.service import com.tencent.devops.common.api.annotation.ServiceInterface +import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_OS_ARCH +import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_OS_NAME +import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_SHA_CONTENT import com.tencent.devops.common.api.pojo.Result +import io.swagger.v3.oas.annotations.Parameter import javax.ws.rs.Consumes import javax.ws.rs.GET +import javax.ws.rs.HeaderParam import javax.ws.rs.Path import javax.ws.rs.PathParam import javax.ws.rs.Produces +import javax.ws.rs.QueryParam import javax.ws.rs.core.MediaType /** - * 验证插件是否有调用敏感接口的权限,在store中实现 + * 验证组件是否有调用敏感接口的权限,在store中实现 */ @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) @@ -46,17 +52,40 @@ import javax.ws.rs.core.MediaType interface ServiceSensitiveApiPermissionResource { /** - * 验证api是否已经审批 + * 验证组件是否有该api接口的权限 * - * @param atomCode 组件编码 + * @param installedPkgShaContent 已安装组件包sha1摘要值 + * @param osName 操作系统名称 + * @param osArch 操作系统CPU架构 + * @param storeCode 组件标识 * @param apiName api接口名称 + * @param storeType 组件类型 + * @param version 组件版本 */ - @Path("verify/{atomCode}/{apiName}") + @Path("verify/{storeCode}/{apiName}") @GET + @Suppress("LongParameterList") fun verifyApi( - @PathParam("atomCode") - atomCode: String, + @Parameter(description = "已安装组件包sha1摘要值", required = false) + @HeaderParam(AUTH_HEADER_DEVOPS_SHA_CONTENT) + installedPkgShaContent: String? = null, + @Parameter(description = "操作系统名称", required = false) + @HeaderParam(AUTH_HEADER_DEVOPS_OS_NAME) + osName: String? = null, + @Parameter(description = "操作系统CPU架构", required = false) + @HeaderParam(AUTH_HEADER_DEVOPS_OS_ARCH) + osArch: String? = null, + @PathParam("storeCode") + @Parameter(description = "组件标识", required = true) + storeCode: String, @PathParam("apiName") - apiName: String + @Parameter(description = "api接口名称", required = true) + apiName: String, + @QueryParam("storeType") + @Parameter(description = "组件类型", required = true) + storeType: String = "ATOM", + @QueryParam("version") + @Parameter(description = "组件版本", required = false) + version: String? = null ): Result } diff --git a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/filter/TokenCheckFilter.kt b/src/backend/ci/core/common/common-web/src/main/kotlin/com/tencent/devops/common/web/service/impl/BkApiHandleOpenAccessServiceImpl.kt similarity index 53% rename from src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/filter/TokenCheckFilter.kt rename to src/backend/ci/core/common/common-web/src/main/kotlin/com/tencent/devops/common/web/service/impl/BkApiHandleOpenAccessServiceImpl.kt index c290ce44df7..f2dfa16a8fc 100644 --- a/src/backend/ci/core/auth/biz-auth/src/main/kotlin/com/tencent/devops/auth/filter/TokenCheckFilter.kt +++ b/src/backend/ci/core/common/common-web/src/main/kotlin/com/tencent/devops/common/web/service/impl/BkApiHandleOpenAccessServiceImpl.kt @@ -25,56 +25,31 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.auth.filter +package com.tencent.devops.common.web.service.impl import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_BK_TOKEN import com.tencent.devops.common.api.exception.TokenForbiddenException import com.tencent.devops.common.client.ClientTokenService +import com.tencent.devops.common.service.utils.SpringContextUtil +import com.tencent.devops.common.web.service.BkApiHandleService import org.slf4j.LoggerFactory -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.stereotype.Component -import javax.servlet.Filter -import javax.servlet.FilterChain -import javax.servlet.FilterConfig -import javax.servlet.ServletRequest -import javax.servlet.ServletResponse -import javax.servlet.http.HttpServletRequest +import org.springframework.web.context.request.RequestContextHolder +import org.springframework.web.context.request.ServletRequestAttributes -@Component -class TokenCheckFilter @Autowired constructor( - val clientTokenService: ClientTokenService -) : Filter { - override fun destroy() = Unit +class BkApiHandleOpenAccessServiceImpl : BkApiHandleService { - override fun doFilter(request: ServletRequest?, response: ServletResponse?, chain: FilterChain?) { - if (request == null || chain == null) { - return - } - val httpServletRequest = request as HttpServletRequest - val token = httpServletRequest.getHeader(AUTH_HEADER_DEVOPS_BK_TOKEN) - - val pathInfo = httpServletRequest.pathInfo ?: return chain.doFilter(request, response) - - if (!pathInfo.contains("/open/")) { - return chain.doFilter(request, response) - } - - // TODO: 配置化, 用于一些信任的第三方调用过来的特殊请求 - if (pathInfo.contains("open/auth/resource/projects") || - pathInfo.contains("open/auth/resource/instances/list")) { - return chain.doFilter(request, response) - } + companion object { + private val logger = LoggerFactory.getLogger(BkApiHandleOpenAccessServiceImpl::class.java) + } - if (token != clientTokenService.getSystemToken()) { - logger.warn("auth token fail: $token") + override fun handleBuildApiService(parameterNames: Array, parameterValue: Array) { + val attributes = RequestContextHolder.getRequestAttributes() as? ServletRequestAttributes ?: return + val request = attributes.request + val token = request.getHeader(AUTH_HEADER_DEVOPS_BK_TOKEN) + val tokenService: ClientTokenService = SpringContextUtil.getBean(ClientTokenService::class.java) + if (token != tokenService.getSystemToken()) { + logger.warn("auth token fail: $token, url: ${request.requestURI}") throw TokenForbiddenException("token check fail") } - chain.doFilter(request, response) - } - - override fun init(filterConfig: FilterConfig?) = Unit - - companion object { - private val logger = LoggerFactory.getLogger(TokenCheckFilter::class.java) } } diff --git a/src/backend/ci/core/common/common-web/src/main/kotlin/com/tencent/devops/common/web/utils/I18nUtil.kt b/src/backend/ci/core/common/common-web/src/main/kotlin/com/tencent/devops/common/web/utils/I18nUtil.kt index 587d7789f91..f8630486012 100644 --- a/src/backend/ci/core/common/common-web/src/main/kotlin/com/tencent/devops/common/web/utils/I18nUtil.kt +++ b/src/backend/ci/core/common/common-web/src/main/kotlin/com/tencent/devops/common/web/utils/I18nUtil.kt @@ -103,6 +103,22 @@ object I18nUtil { } } + /** + * 获取蓝鲸标准的语言Header Key + */ + fun getBKLanguageKey(): String { + return BK_LANGUAGE + } + + /** + * 获取Http Cookie中用户携带的语言（蓝鲸标准） + * @return 语言,如果没有,则为空 + */ + fun getBKLanguageFromCookie(): String? { + val request = BkApiUtil.getHttpServletRequest() + return request?.let { CookieUtil.getCookieValue(request, BK_LANGUAGE) } + } + /** * 获取Http Cookie中用户携带的语言 * @return 语言,如果没有,则为空 diff --git a/src/backend/ci/core/common/common-webhook/api-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/enums/WebhookI18nConstants.kt b/src/backend/ci/core/common/common-webhook/api-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/enums/WebhookI18nConstants.kt index e6fdea63e4b..4dcbdcdf679 100644 --- a/src/backend/ci/core/common/common-webhook/api-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/enums/WebhookI18nConstants.kt +++ b/src/backend/ci/core/common/common-webhook/api-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/enums/WebhookI18nConstants.kt @@ -84,6 +84,12 @@ object WebhookI18nConstants { // 事件类型不匹配 const val EVENT_TYPE_NOT_MATCH = "bkRepoTriggerEventTypeNotMatch" + // 分支不匹配 + const val BRANCH_NOT_MATCH = "bkRepoTriggerBranchNotMatch" + + // 分支被排除 + const val BRANCH_IGNORED = "bkRepoTriggerBranchIgnored" + // 目标分支不匹配 const val TARGET_BRANCH_NOT_MATCH = "bkRepoTriggerTargetBranchNotMatch" diff --git a/src/backend/ci/core/common/common-webhook/api-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/pojo/code/svn/SvnCommitEvent.kt b/src/backend/ci/core/common/common-webhook/api-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/pojo/code/svn/SvnCommitEvent.kt index fa542d74ad4..84df10e0384 100644 --- a/src/backend/ci/core/common/common-webhook/api-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/pojo/code/svn/SvnCommitEvent.kt +++ b/src/backend/ci/core/common/common-webhook/api-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/pojo/code/svn/SvnCommitEvent.kt @@ -28,6 +28,7 @@ package com.tencent.devops.common.webhook.pojo.code.svn import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.annotation.JsonProperty import com.tencent.devops.common.webhook.pojo.code.CodeWebhookEvent @Suppress("ALL") @@ -40,5 +41,7 @@ data class SvnCommitEvent( val revision: Int, val paths: List, val files: List, - val commitTime: Long? + val commitTime: Long?, + @JsonProperty("total_files_count") + val totalFilesCount: Int? ) : CodeWebhookEvent diff --git a/src/backend/ci/core/common/common-webhook/biz-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/service/code/GitScmService.kt b/src/backend/ci/core/common/common-webhook/biz-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/service/code/GitScmService.kt index b328aebd683..8f28471308b 100644 --- a/src/backend/ci/core/common/common-webhook/biz-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/service/code/GitScmService.kt +++ b/src/backend/ci/core/common/common-webhook/biz-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/service/code/GitScmService.kt @@ -37,6 +37,7 @@ import com.tencent.devops.repository.api.github.ServiceGithubPRResource import com.tencent.devops.repository.api.scm.ServiceGitResource import com.tencent.devops.repository.api.scm.ServiceScmOauthResource import com.tencent.devops.repository.api.scm.ServiceScmResource +import com.tencent.devops.repository.api.scm.ServiceTGitResource import com.tencent.devops.repository.pojo.CodeGitRepository import com.tencent.devops.repository.pojo.CodeGitlabRepository import com.tencent.devops.repository.pojo.CodeTGitRepository @@ -212,16 +213,30 @@ class GitScmService @Autowired constructor( ) for (i in 1..10) { // 反向进行三点比较可以比较出rebase的真实提交 - val result = client.get(ServiceGitResource::class).getChangeFileList( - token = token, - tokenType = tokenType, - gitProjectId = repo.projectName, - from = from, - to = to, - straight = false, - page = i, - pageSize = 100 - ).data ?: emptyList() + val result = if (repo.getScmType() == ScmType.CODE_TGIT) { + client.get(ServiceTGitResource::class).getChangeFileList( + token = token, + tokenType = tokenType, + gitProjectId = repo.projectName, + from = from, + to = to, + straight = false, + page = i, + pageSize = 100, + url = repo.url + ).data + } else { + client.get(ServiceGitResource::class).getChangeFileList( + token = token, + tokenType = tokenType, + gitProjectId = repo.projectName, + from = from, + to = to, + straight = false, + page = i, + pageSize = 100 + ).data + } ?: emptyList() changeSet.addAll( result.map { if (it.deletedFile) { diff --git a/src/backend/ci/core/common/common-webhook/biz-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/service/code/handler/svn/SvnCommitTriggerHandler.kt b/src/backend/ci/core/common/common-webhook/biz-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/service/code/handler/svn/SvnCommitTriggerHandler.kt index c13f1b29601..0d606bfc3e0 100644 --- a/src/backend/ci/core/common/common-webhook/biz-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/service/code/handler/svn/SvnCommitTriggerHandler.kt +++ b/src/backend/ci/core/common/common-webhook/biz-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/service/code/handler/svn/SvnCommitTriggerHandler.kt @@ -45,6 +45,7 @@ import com.tencent.devops.common.webhook.service.code.filter.WebhookFilter import com.tencent.devops.common.webhook.service.code.handler.CodeWebhookTriggerHandler import com.tencent.devops.common.webhook.util.WebhookUtils import com.tencent.devops.repository.pojo.Repository +import org.slf4j.LoggerFactory @CodeWebhookHandler @SuppressWarnings("TooManyFunctions") @@ -128,7 +129,7 @@ class SvnCommitTriggerHandler : CodeWebhookTriggerHandler { PathFilterConfig( pathFilterType = pathFilterType, pipelineId = pipelineId, - triggerOnPath = event.files.map { it.file }, + triggerOnPath = event.getMatchPaths(), excludedPaths = WebhookUtils.convert(excludePaths).map { path -> WebhookUtils.getFullPath( projectRelativePath = projectRelativePath, @@ -189,4 +190,18 @@ class SvnCommitTriggerHandler : CodeWebhookTriggerHandler { startParams[BK_REPO_SVN_WEBHOOK_COMMIT_TIME] = event.commitTime ?: 0L return startParams } + + fun SvnCommitEvent.getMatchPaths() = if ((totalFilesCount ?: 0) < FILES_COUNT_MAX) { + files.map { it.file } + } else { + // 超过上限则存在变更记录丢失, 用paths进行匹配 + logger.info("File change information exceeds the limit|$totalFilesCount") + paths + } + + companion object { + // 文件变更列表上限 + const val FILES_COUNT_MAX = 999 + private val logger = LoggerFactory.getLogger(SvnCommitTriggerHandler::class.java) + } } diff --git a/src/backend/ci/core/common/common-webhook/biz-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/service/code/handler/tgit/TGitIssueTriggerHandler.kt b/src/backend/ci/core/common/common-webhook/biz-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/service/code/handler/tgit/TGitIssueTriggerHandler.kt index 0f315180639..b8d00c479c0 100644 --- a/src/backend/ci/core/common/common-webhook/biz-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/service/code/handler/tgit/TGitIssueTriggerHandler.kt +++ b/src/backend/ci/core/common/common-webhook/biz-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/service/code/handler/tgit/TGitIssueTriggerHandler.kt @@ -175,6 +175,7 @@ class TGitIssueTriggerHandler( return listOf(actionFilter) } + @Suppress("ComplexMethod") override fun retrieveParams(event: GitIssueEvent, projectId: String?, repository: Repository?): Map { val startParams = mutableMapOf() with(event.objectAttributes) { diff --git a/src/backend/ci/core/common/common-webhook/biz-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/service/code/handler/tgit/TGitMrTriggerHandler.kt b/src/backend/ci/core/common/common-webhook/biz-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/service/code/handler/tgit/TGitMrTriggerHandler.kt index 57e3a7c9811..18cae3e2895 100644 --- a/src/backend/ci/core/common/common-webhook/biz-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/service/code/handler/tgit/TGitMrTriggerHandler.kt +++ b/src/backend/ci/core/common/common-webhook/biz-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/service/code/handler/tgit/TGitMrTriggerHandler.kt @@ -49,6 +49,8 @@ import com.tencent.devops.common.pipeline.utils.PIPELINE_GIT_MR_URL import com.tencent.devops.common.pipeline.utils.PIPELINE_GIT_REPO_URL import com.tencent.devops.common.webhook.annotation.CodeWebhookHandler import com.tencent.devops.common.webhook.enums.WebhookI18nConstants +import com.tencent.devops.common.webhook.enums.code.tgit.TGitMergeActionKind.UPDATE +import com.tencent.devops.common.webhook.enums.code.tgit.TGitMergeActionKind import com.tencent.devops.common.webhook.enums.code.tgit.TGitMrEventAction import com.tencent.devops.common.webhook.pojo.code.BK_REPO_GIT_MANUAL_UNLOCK import com.tencent.devops.common.webhook.pojo.code.BK_REPO_GIT_WEBHOOK_BRANCH @@ -242,7 +244,11 @@ class TGitMrTriggerHandler( val actionFilter = ContainsFilter( pipelineId = pipelineId, filterName = "mrAction", - triggerOn = TGitMrEventAction.getActionValue(event) ?: "", + triggerOn = if (repository is CodeGitlabRepository && getAction(event) == UPDATE.value) { + TGitMrEventAction.PUSH_UPDATE.value + } else { + TGitMrEventAction.getActionValue(event) + } ?: "", included = convert(includeMrAction).ifEmpty { listOf("empty-action") }, @@ -359,7 +365,11 @@ class TGitMrTriggerHandler( startParams[PIPELINE_GIT_MR_ACTION] = event.object_attributes.action ?: "" startParams[PIPELINE_GIT_ACTION] = event.object_attributes.action ?: "" startParams[PIPELINE_GIT_EVENT_URL] = event.object_attributes.url ?: "" - startParams[BK_REPO_GIT_WEBHOOK_BRANCH] = event.object_attributes.source_branch + startParams[BK_REPO_GIT_WEBHOOK_BRANCH] = if (getAction(event) == TGitMergeActionKind.MERGE.value) { + event.object_attributes.target_branch + } else { + event.object_attributes.source_branch + } // 有覆盖风险的上下文做二次确认 startParams.putIfEmpty(GIT_MR_NUMBER, event.object_attributes.iid.toString()) startParams.putIfEmpty(PIPELINE_GIT_MR_ID, event.object_attributes.id.toString()) diff --git a/src/backend/ci/core/common/common-webhook/biz-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/service/code/handler/tgit/TGitPushTriggerHandler.kt b/src/backend/ci/core/common/common-webhook/biz-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/service/code/handler/tgit/TGitPushTriggerHandler.kt index a0aff6892b8..bd2d273da4a 100644 --- a/src/backend/ci/core/common/common-webhook/biz-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/service/code/handler/tgit/TGitPushTriggerHandler.kt +++ b/src/backend/ci/core/common/common-webhook/biz-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/service/code/handler/tgit/TGitPushTriggerHandler.kt @@ -96,6 +96,10 @@ class TGitPushTriggerHandler( companion object { private val logger = LoggerFactory.getLogger(TGitPushTriggerHandler::class.java) + // 空提交点，可用于推断是新增/删除分支 + // 新增分支 -> before为此值 + // 删除分支 -> after为此值 + const val EMPTY_COMMIT_ID = "0000000000000000000000000000000000000000" } override fun eventClass(): Class { @@ -135,7 +139,11 @@ class TGitPushTriggerHandler( } override fun getAction(event: GitPushEvent): String? { - return event.action_kind + return when { + event.action_kind.isNullOrBlank() -> event.action_kind + event.before == EMPTY_COMMIT_ID -> TGitPushActionType.NEW_BRANCH.value + else -> TGitPushActionType.PUSH_FILE.value + } } override fun getEventDesc(event: GitPushEvent): String { @@ -200,11 +208,11 @@ class TGitPushTriggerHandler( includedBranches = convert(branchName), excludedBranches = convert(excludeBranchName), includedFailedReason = I18Variable( - code = WebhookI18nConstants.SOURCE_BRANCH_NOT_MATCH, + code = WebhookI18nConstants.BRANCH_NOT_MATCH, params = listOf(triggerOnBranchName) ).toJsonStr(), excludedFailedReason = I18Variable( - code = WebhookI18nConstants.SOURCE_BRANCH_IGNORED, + code = WebhookI18nConstants.BRANCH_IGNORED, params = listOf(triggerOnBranchName) ).toJsonStr() ) diff --git a/src/backend/ci/core/common/common-webhook/biz-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/service/code/param/GitWebhookElementParams.kt b/src/backend/ci/core/common/common-webhook/biz-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/service/code/param/GitWebhookElementParams.kt index 4975a48250f..d952bc679ae 100644 --- a/src/backend/ci/core/common/common-webhook/biz-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/service/code/param/GitWebhookElementParams.kt +++ b/src/backend/ci/core/common/common-webhook/biz-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/service/code/param/GitWebhookElementParams.kt @@ -76,7 +76,7 @@ class GitWebhookElementParams : ScmWebhookElementParams { - params.includeMrAction = joinToString( + params.includeMrAction = WebhookUtils.joinToString( listOf( CodeGitWebHookTriggerElement.MERGE_ACTION_OPEN, CodeGitWebHookTriggerElement.MERGE_ACTION_REOPEN, @@ -88,7 +88,7 @@ class GitWebhookElementParams : ScmWebhookElementParams { - params.includePushAction = joinToString( + params.includePushAction = WebhookUtils.joinToString( listOf( CodeGitWebHookTriggerElement.PUSH_ACTION_CREATE_BRANCH, CodeGitWebHookTriggerElement.PUSH_ACTION_PUSH_FILE @@ -97,8 +97,8 @@ class GitWebhookElementParams : ScmWebhookElementParams { - params.includeMrAction = joinToString(element.includeMrAction) - params.includePushAction = joinToString(element.includePushAction) + params.includeMrAction = WebhookUtils.joinToString(element.includeMrAction) + params.includePushAction = WebhookUtils.joinToString(element.includePushAction) } } params.eventType = element.eventType @@ -112,11 +112,11 @@ class GitWebhookElementParams : ScmWebhookElementParams?): String { - return if (list.isNullOrEmpty()) { - "" - } else { - list.joinToString(",") - } - } - private fun isBlock(element: CodeGitWebHookTriggerElement): Boolean { return when { element.enableCheck == false || element.eventType != CodeEventType.MERGE_REQUEST -> false diff --git a/src/backend/ci/core/common/common-webhook/biz-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/service/code/param/GitlabWebhookElementParams.kt b/src/backend/ci/core/common/common-webhook/biz-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/service/code/param/GitlabWebhookElementParams.kt index cd82a08d8eb..93dcbb7b1ff 100644 --- a/src/backend/ci/core/common/common-webhook/biz-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/service/code/param/GitlabWebhookElementParams.kt +++ b/src/backend/ci/core/common/common-webhook/biz-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/service/code/param/GitlabWebhookElementParams.kt @@ -28,10 +28,13 @@ package com.tencent.devops.common.webhook.service.code.param import com.tencent.devops.common.api.util.EnvUtils +import com.tencent.devops.common.pipeline.pojo.element.trigger.CodeGitWebHookTriggerElement import com.tencent.devops.common.pipeline.pojo.element.trigger.CodeGitlabWebHookTriggerElement +import com.tencent.devops.common.pipeline.pojo.element.trigger.enums.CodeEventType import com.tencent.devops.common.pipeline.pojo.element.trigger.enums.CodeType import com.tencent.devops.common.pipeline.utils.RepositoryConfigUtils import com.tencent.devops.common.webhook.pojo.code.WebHookParams +import com.tencent.devops.common.webhook.util.WebhookUtils import org.springframework.stereotype.Service @Service @@ -62,10 +65,42 @@ class GitlabWebhookElementParams : ScmWebhookElementParams { + params.includeMrAction = CodeGitWebHookTriggerElement.MERGE_ACTION_MERGE + } + + element.eventType == CodeEventType.MERGE_REQUEST && + !WebhookUtils.isActionGitTriggerVersion(element.version) && + element.includeMrAction == null -> { + params.includeMrAction = WebhookUtils.joinToString( + listOf( + CodeGitWebHookTriggerElement.MERGE_ACTION_OPEN, + CodeGitWebHookTriggerElement.MERGE_ACTION_REOPEN, + CodeGitWebHookTriggerElement.MERGE_ACTION_PUSH_UPDATE + ) + ) + } + + element.eventType == CodeEventType.PUSH && + !WebhookUtils.isActionGitTriggerVersion(element.version) && + element.includePushAction == null -> { + params.includePushAction = WebhookUtils.joinToString( + listOf( + CodeGitWebHookTriggerElement.PUSH_ACTION_CREATE_BRANCH, + CodeGitWebHookTriggerElement.PUSH_ACTION_PUSH_FILE + ) + ) + } + + else -> { + params.includeMrAction = WebhookUtils.joinToString(element.includeMrAction) + params.includePushAction = WebhookUtils.joinToString(element.includePushAction) + } } - params.branchName = EnvUtils.parseEnv(element.branchName!!, variables) + params.branchName = EnvUtils.parseEnv(element.branchName ?: "", variables) params.codeType = CodeType.GITLAB params.eventType = element.eventType params.block = element.block ?: false diff --git a/src/backend/ci/core/common/common-webhook/biz-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/util/WebhookUtils.kt b/src/backend/ci/core/common/common-webhook/biz-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/util/WebhookUtils.kt index 4ef086b81e4..9fceb0c9d10 100644 --- a/src/backend/ci/core/common/common-webhook/biz-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/util/WebhookUtils.kt +++ b/src/backend/ci/core/common/common-webhook/biz-common-webhook/src/main/kotlin/com/tencent/devops/common/webhook/util/WebhookUtils.kt @@ -356,4 +356,12 @@ object WebhookUtils { )) return startParams } + + fun joinToString(list: List?): String { + return if (list.isNullOrEmpty()) { + "" + } else { + list.joinToString(",") + } + } } diff --git a/src/backend/ci/core/dispatch/api-dispatch-kubernetes/build.gradle.kts b/src/backend/ci/core/dispatch/api-dispatch-kubernetes/build.gradle.kts index d6787a1f571..d696b7475a9 100644 --- a/src/backend/ci/core/dispatch/api-dispatch-kubernetes/build.gradle.kts +++ b/src/backend/ci/core/dispatch/api-dispatch-kubernetes/build.gradle.kts @@ -29,6 +29,7 @@ dependencies { api(project(":core:common:common-api")) api(project(":core:common:common-web")) api(project(":core:buildless:api-buildless")) + api("io.fabric8:kubernetes-client") } plugins { diff --git a/src/backend/ci/core/dispatch/api-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/api/service/ServiceDockerImageResource.kt b/src/backend/ci/core/dispatch/api-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/api/service/ServiceDockerImageResource.kt new file mode 100644 index 00000000000..743c152ec85 --- /dev/null +++ b/src/backend/ci/core/dispatch/api-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/api/service/ServiceDockerImageResource.kt @@ -0,0 +1,63 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.dispatch.kubernetes.api.service + +import com.tencent.devops.common.api.annotation.ServiceInterface +import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID +import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID_DEFAULT_VALUE +import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.dispatch.kubernetes.pojo.CheckDockerImageRequest +import com.tencent.devops.dispatch.kubernetes.pojo.CheckDockerImageResponse +import io.swagger.v3.oas.annotations.Operation +import io.swagger.v3.oas.annotations.Parameter +import io.swagger.v3.oas.annotations.tags.Tag +import javax.ws.rs.Consumes +import javax.ws.rs.HeaderParam +import javax.ws.rs.POST +import javax.ws.rs.Path +import javax.ws.rs.Produces +import javax.ws.rs.core.MediaType + +@Tag(name = "SERVICE_DOCKER_IMAGE", description = "镜像-镜像服务") +@Path("/service/docker-image") +@ServiceInterface("dispatch") // 指明接入到哪个微服务 +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +interface ServiceDockerImageResource { + + @POST + @Path("/checkDockerImage") + @Operation(summary = "检查镜像信息") + fun checkDockerImage( + @Parameter(description = "用户ID", required = true, example = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @Parameter(description = "镜像repo", required = true) + checkDockerImageRequestList: List + ): Result> +} diff --git a/src/backend/ci/core/dispatch/api-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/pojo/CheckDockerImageRequest.kt b/src/backend/ci/core/dispatch/api-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/pojo/CheckDockerImageRequest.kt new file mode 100644 index 00000000000..34967482ed3 --- /dev/null +++ b/src/backend/ci/core/dispatch/api-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/pojo/CheckDockerImageRequest.kt @@ -0,0 +1,41 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.dispatch.kubernetes.pojo + +import io.swagger.v3.oas.annotations.media.Schema + +data class CheckDockerImageRequest( + @get:Schema(title = "镜像名称", required = true) + val imageName: String, + @get:Schema(title = "镜像仓库", required = true) + val registryHost: String, + @get:Schema(title = "用户名", required = false) + val registryUser: String?, + @get:Schema(title = "密码", required = false) + val registryPwd: String? +) diff --git a/src/backend/ci/core/dispatch/api-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/pojo/CheckDockerImageResponse.kt b/src/backend/ci/core/dispatch/api-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/pojo/CheckDockerImageResponse.kt new file mode 100644 index 00000000000..111b7791f22 --- /dev/null +++ b/src/backend/ci/core/dispatch/api-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/pojo/CheckDockerImageResponse.kt @@ -0,0 +1,66 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.dispatch.kubernetes.pojo + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import io.swagger.v3.oas.annotations.media.Schema + +@JsonIgnoreProperties(ignoreUnknown = true) +@Schema(title = "检查镜像信息返回模型") +data class CheckDockerImageResponse( + @get:Schema(title = "错误代码") + val errorCode: Int, + @get:Schema(title = "错误信息") + val errorMessage: String? = "", + @get:Schema(title = "架构") + val arch: String? = "", + @get:Schema(title = "作者") + val author: String? = "", + @get:Schema(title = "评论") + val comment: String? = "", + @get:Schema(title = "创建时间") + val created: String? = "", + @get:Schema(title = "docker版本") + val dockerVersion: String? = "", + @get:Schema(title = "id") + val id: String? = "", + @get:Schema(title = "操作系统") + val os: String? = "", + @get:Schema(title = "操作系统版本") + val osVersion: String? = "", + @get:Schema(title = "父容器") + val parent: String? = "", + @get:Schema(title = "大小") + val size: Long? = 0, + @get:Schema(title = "仓库标签") + val repoTags: List? = null, + @get:Schema(title = "image存储属性") + val repoDigests: List? = null, + @get:Schema(title = "虚拟大小") + val virtualSize: Long? = 0 +) diff --git a/src/backend/ci/core/dispatch/api-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/pojo/base/KubernetesRepo.kt b/src/backend/ci/core/dispatch/api-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/pojo/base/KubernetesRepo.kt new file mode 100644 index 00000000000..83c5a60f1ba --- /dev/null +++ b/src/backend/ci/core/dispatch/api-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/pojo/base/KubernetesRepo.kt @@ -0,0 +1,42 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.dispatch.kubernetes.pojo.base + +import io.swagger.v3.oas.annotations.media.Schema + +@Schema(title = "k8s仓库信息") +data class KubernetesRepo( + @get:Schema(title = "仓库地址", required = true) + val registryUrl: String, + @get:Schema(title = "用户名", required = true) + val username: String, + @get:Schema(title = "密码", required = true) + val password: String, + @get:Schema(title = "邮箱", required = false) + val email: String? +) diff --git a/src/backend/ci/core/dispatch/api-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/pojo/common/ErrorCodeEnum.kt b/src/backend/ci/core/dispatch/api-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/pojo/common/ErrorCodeEnum.kt index 5f7b282b3fd..ca84eb0f29f 100644 --- a/src/backend/ci/core/dispatch/api-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/pojo/common/ErrorCodeEnum.kt +++ b/src/backend/ci/core/dispatch/api-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/pojo/common/ErrorCodeEnum.kt @@ -303,6 +303,11 @@ enum class ErrorCodeEnum( ErrorType.USER, 2126054, "已超过DevCloud创建Job环境上限." + ), + KUBERNETES_CREATE_RESOURCE_FAIL( + ErrorType.SYSTEM, + 2126055, + "kubernetes创建{0}资源失败，原因：{1}" ); fun getErrorMessage(): String { diff --git a/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/bcs/service/BcsContainerService.kt b/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/bcs/service/BcsContainerService.kt index 0461b662af0..4b657fbe8e9 100644 --- a/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/bcs/service/BcsContainerService.kt +++ b/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/bcs/service/BcsContainerService.kt @@ -313,7 +313,7 @@ class BcsContainerService @Autowired constructor( } } - override fun waitTaskFinish(userId: String, taskId: String): DispatchBuildTaskStatus { + override fun waitTaskFinish(userId: String, taskId: String, needProxy: Boolean): DispatchBuildTaskStatus { val startResult = bcsTaskClient.waitTaskFinish(userId, taskId) if (startResult.first == BcsTaskStatusEnum.SUCCEEDED) { return DispatchBuildTaskStatus(DispatchBuildTaskStatusEnum.SUCCEEDED, null) @@ -395,4 +395,8 @@ class BcsContainerService @Autowired constructor( ) ) } + + override fun inspectDockerImage(userId: String, pool: Pool): String { + return "" + } } diff --git a/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/client/DeploymentClient.kt b/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/client/DeploymentClient.kt new file mode 100644 index 00000000000..6eb1cf4505c --- /dev/null +++ b/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/client/DeploymentClient.kt @@ -0,0 +1,116 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.dispatch.kubernetes.client + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue +import com.tencent.devops.common.api.util.JsonUtil +import com.tencent.devops.common.api.util.OkhttpUtils +import com.tencent.devops.common.dispatch.sdk.BuildFailureException +import com.tencent.devops.dispatch.kubernetes.pojo.KubernetesResult +import com.tencent.devops.dispatch.kubernetes.pojo.common.ErrorCodeEnum +import io.fabric8.kubernetes.api.model.apps.Deployment +import okhttp3.MediaType.Companion.toMediaTypeOrNull +import okhttp3.RequestBody +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired + +class DeploymentClient @Autowired constructor( + private val objectMapper: ObjectMapper, + private val clientCommon: KubernetesClientCommon +) { + + companion object { + private val logger = LoggerFactory.getLogger(DeploymentClient::class.java) + } + + fun createDeployment( + userId: String, + namespace: String, + deployment: Deployment + ): KubernetesResult { + val url = "/api/namespace/$namespace/deployments" + val body = JsonUtil.toJson(deployment) + logger.info("Create deployment request url: $url, body: $body") + val request = clientCommon.microBaseRequest(url).post( + RequestBody.create( + "application/json; charset=utf-8".toMediaTypeOrNull(), + body + ) + ).build() + val responseBody = OkhttpUtils.doHttp(request).body!!.string() + logger.info("Create deployment response: ${JsonUtil.toJson(responseBody)}") + return JsonUtil.getObjectMapper().readValue(responseBody) + } + + fun getDeploymentByName( + userId: String, + namespace: String, + deploymentName: String + ): KubernetesResult { + val url = "/api/namespace/$namespace/deployments/$deploymentName" + val request = clientCommon.microBaseRequest(url).get().build() + logger.info("Get deployment: $deploymentName request url: $url, userId: $userId") + OkhttpUtils.doHttp(request).use { response -> + val responseContent = response.body!!.string() + logger.info("Get deployment: $deploymentName response: $responseContent") + if (!response.isSuccessful) { + throw BuildFailureException( + errorType = ErrorCodeEnum.BCS_SYSTEM_ERROR.errorType, + errorCode = ErrorCodeEnum.BCS_SYSTEM_ERROR.errorCode, + formatErrorMessage = ErrorCodeEnum.BCS_SYSTEM_ERROR.getErrorMessage(), + errorMessage = "Fail to get deployment,http response code: ${response.code}" + ) + } + return objectMapper.readValue(responseContent) + } + } + + fun deleteDeploymentByName( + userId: String, + namespace: String, + deploymentName: String + ): KubernetesResult { + val url = "/api/namespace/$namespace/deployments/$deploymentName" + val request = clientCommon.microBaseRequest(url).delete().build() + logger.info("Delete deployment: $deploymentName request url: $url, userId: $userId") + OkhttpUtils.doHttp(request).use { response -> + val responseContent = response.body!!.string() + logger.info("Delete deployment: $deploymentName response: $responseContent") + if (!response.isSuccessful) { + throw BuildFailureException( + errorType = ErrorCodeEnum.BCS_SYSTEM_ERROR.errorType, + errorCode = ErrorCodeEnum.BCS_SYSTEM_ERROR.errorCode, + formatErrorMessage = ErrorCodeEnum.BCS_SYSTEM_ERROR.getErrorMessage(), + errorMessage = "Fail to delete deployment,http response code: ${response.code}" + ) + } + return objectMapper.readValue(responseContent) + } + } +} diff --git a/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/client/IngressClient.kt b/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/client/IngressClient.kt new file mode 100644 index 00000000000..9cf45037004 --- /dev/null +++ b/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/client/IngressClient.kt @@ -0,0 +1,116 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.dispatch.kubernetes.client + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue +import com.tencent.devops.common.api.util.JsonUtil +import com.tencent.devops.common.api.util.OkhttpUtils +import com.tencent.devops.common.dispatch.sdk.BuildFailureException +import com.tencent.devops.dispatch.kubernetes.pojo.KubernetesResult +import com.tencent.devops.dispatch.kubernetes.pojo.common.ErrorCodeEnum +import io.fabric8.kubernetes.api.model.networking.v1.Ingress +import okhttp3.MediaType.Companion.toMediaTypeOrNull +import okhttp3.RequestBody +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired + +class IngressClient @Autowired constructor( + private val objectMapper: ObjectMapper, + private val clientCommon: KubernetesClientCommon +) { + + companion object { + private val logger = LoggerFactory.getLogger(IngressClient::class.java) + } + + fun createIngress( + userId: String, + namespace: String, + ingress: Ingress + ): KubernetesResult { + val url = "/api/namespace/$namespace/ingress" + val body = JsonUtil.toJson(ingress) + logger.info("Create ingress request url: $url, body: $body") + val request = clientCommon.microBaseRequest(url).post( + RequestBody.create( + "application/json; charset=utf-8".toMediaTypeOrNull(), + body + ) + ).build() + val responseBody = OkhttpUtils.doHttp(request).body!!.string() + logger.info("Create ingress response: ${JsonUtil.toJson(responseBody)}") + return JsonUtil.getObjectMapper().readValue(responseBody) + } + + fun getIngressByName( + userId: String, + namespace: String, + ingressName: String + ): KubernetesResult { + val url = "/api/namespace/$namespace/ingress/$ingressName" + val request = clientCommon.microBaseRequest(url).get().build() + logger.info("Get ingress: $ingressName request url: $url, userId: $userId") + OkhttpUtils.doHttp(request).use { response -> + val responseContent = response.body!!.string() + logger.info("Get ingress: $ingressName response: $responseContent") + if (!response.isSuccessful) { + throw BuildFailureException( + errorType = ErrorCodeEnum.BCS_SYSTEM_ERROR.errorType, + errorCode = ErrorCodeEnum.BCS_SYSTEM_ERROR.errorCode, + formatErrorMessage = ErrorCodeEnum.BCS_SYSTEM_ERROR.getErrorMessage(), + errorMessage = "Fail to get ingress,http response code: ${response.code}" + ) + } + return objectMapper.readValue(responseContent) + } + } + + fun deleteIngressByName( + userId: String, + namespace: String, + ingressName: String + ): KubernetesResult { + val url = "/api/namespace/$namespace/ingress/$ingressName" + val request = clientCommon.microBaseRequest(url).delete().build() + logger.info("Delete ingress: $ingressName request url: $url, userId: $userId") + OkhttpUtils.doHttp(request).use { response -> + val responseContent = response.body!!.string() + logger.info("Delete ingress: $ingressName response: $responseContent") + if (!response.isSuccessful) { + throw BuildFailureException( + errorType = ErrorCodeEnum.BCS_SYSTEM_ERROR.errorType, + errorCode = ErrorCodeEnum.BCS_SYSTEM_ERROR.errorCode, + formatErrorMessage = ErrorCodeEnum.BCS_SYSTEM_ERROR.getErrorMessage(), + errorMessage = "Fail to delete ingress,http response code: ${response.code}" + ) + } + return objectMapper.readValue(responseContent) + } + } +} diff --git a/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/client/KubernetesBuilderClient.kt b/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/client/KubernetesBuilderClient.kt index 32016c406b1..b8ee7e906fa 100644 --- a/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/client/KubernetesBuilderClient.kt +++ b/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/client/KubernetesBuilderClient.kt @@ -30,10 +30,14 @@ package com.tencent.devops.dispatch.kubernetes.client import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue +import com.tencent.devops.common.api.util.JsonUtil import com.tencent.devops.common.api.util.OkhttpUtils import com.tencent.devops.common.dispatch.sdk.BuildFailureException import com.tencent.devops.dispatch.kubernetes.pojo.Builder +import com.tencent.devops.dispatch.kubernetes.pojo.Credential import com.tencent.devops.dispatch.kubernetes.pojo.DeleteBuilderParams +import com.tencent.devops.dispatch.kubernetes.pojo.InspectImageCredential +import com.tencent.devops.dispatch.kubernetes.pojo.InspectImageReq import com.tencent.devops.dispatch.kubernetes.pojo.KubernetesBuilderStatus import com.tencent.devops.dispatch.kubernetes.pojo.KubernetesBuilderStatusEnum import com.tencent.devops.dispatch.kubernetes.pojo.KubernetesResult @@ -46,6 +50,7 @@ import com.tencent.devops.dispatch.kubernetes.pojo.getCodeMessage import com.tencent.devops.dispatch.kubernetes.pojo.isRunning import okhttp3.MediaType.Companion.toMediaTypeOrNull import okhttp3.RequestBody +import okhttp3.RequestBody.Companion.toRequestBody import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import java.net.SocketTimeoutException @@ -318,4 +323,57 @@ class KubernetesBuilderClient @Autowired constructor( ) } } + + fun inspectDockerImage( + userId: String, + imageName: String, + credential: Credential + ): String { + val url = "/api/docker/inspect" + val body = InspectImageReq( + name = "${System.currentTimeMillis()}", + ref = imageName, + cred = InspectImageCredential( + username = credential.user, + password = credential.password + ) + ) + + val request = clientCommon.microBaseRequest(url).post(JsonUtil.toJson(body).toRequestBody()).build() + logger.info("$userId inspectImage: $imageName request url: $url, body: $body") + try { + OkhttpUtils.doHttp(request).use { response -> + val responseContent = response.body!!.string() + if (!response.isSuccessful) { + throw BuildFailureException( + errorType = ErrorCodeEnum.BCS_OPERATE_VM_INTERFACE_ERROR.errorType, + errorCode = ErrorCodeEnum.BCS_OPERATE_VM_INTERFACE_ERROR.errorCode, + formatErrorMessage = ErrorCodeEnum.BCS_OPERATE_VM_INTERFACE_ERROR.getErrorMessage(), + errorMessage = "Fail to inspect image, ResponseCode: ${response.code}" + ) + } + logger.info("$userId inspect image: $imageName response: $responseContent") + val responseData: KubernetesResult = objectMapper.readValue(responseContent) + if (responseData.isOk()) { + return responseData.data!!.taskId + } else { + val msg = "${responseData.message ?: responseData.getCodeMessage()}" + throw BuildFailureException( + errorType = ErrorCodeEnum.BCS_OPERATE_VM_INTERFACE_FAIL.errorType, + errorCode = ErrorCodeEnum.BCS_OPERATE_VM_INTERFACE_FAIL.errorCode, + formatErrorMessage = ErrorCodeEnum.BCS_OPERATE_VM_INTERFACE_FAIL.getErrorMessage(), + errorMessage = "Inspect image interface returns a failure：$msg" + ) + } + } + } catch (e: SocketTimeoutException) { + logger.error("$userId inspect image get SocketTimeoutException.", e) + throw BuildFailureException( + errorType = ErrorCodeEnum.BCS_OPERATE_VM_INTERFACE_FAIL.errorType, + errorCode = ErrorCodeEnum.BCS_OPERATE_VM_INTERFACE_FAIL.errorCode, + formatErrorMessage = ErrorCodeEnum.BCS_OPERATE_VM_INTERFACE_FAIL.getErrorMessage(), + errorMessage = "Inspect image interface timed out, url: $url" + ) + } + } } diff --git a/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/client/KubernetesClientCommon.kt b/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/client/KubernetesClientCommon.kt index 9ccfc7b3f22..65ae68fe873 100644 --- a/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/client/KubernetesClientCommon.kt +++ b/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/client/KubernetesClientCommon.kt @@ -52,6 +52,10 @@ class KubernetesClientCommon @Autowired constructor( return Request.Builder().url(commonService.getProxyUrl(kubernetesApiUrl + url)).headers(headers(headers)) } + fun microBaseRequest(url: String, headers: Map? = null): Request.Builder { + return Request.Builder().url(kubernetesApiUrl + url).headers(headers(headers)) + } + fun headers(otherHeaders: Map? = null): Headers { val result = mutableMapOf() diff --git a/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/client/KubernetesTaskClient.kt b/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/client/KubernetesTaskClient.kt index 2f9559856c3..04dfb49b75a 100644 --- a/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/client/KubernetesTaskClient.kt +++ b/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/client/KubernetesTaskClient.kt @@ -60,10 +60,16 @@ class KubernetesTaskClient @Autowired constructor( fun getTasksStatus( userId: String, taskId: String, - retryFlag: Int = 3 + retryFlag: Int = 3, + needProxy: Boolean = true ): KubernetesResult { val url = "/api/tasks/$taskId/status" - val request = clientCommon.baseRequest(userId, url).get().build() + val request = if (needProxy) { + clientCommon.baseRequest(userId, url).get().build() + } else { + clientCommon.microBaseRequest(url).get().build() + } + try { OkhttpUtils.doHttp(request).use { response -> val responseContent = response.body!!.string() @@ -100,7 +106,7 @@ class KubernetesTaskClient @Autowired constructor( } } - fun waitTaskFinish(userId: String, taskId: String): Pair { + fun waitTaskFinish(userId: String, taskId: String, needProxy: Boolean): Pair { val startTime = System.currentTimeMillis() loop@ while (true) { if (System.currentTimeMillis() - startTime > 10 * 60 * 1000) { @@ -111,7 +117,7 @@ class KubernetesTaskClient @Autowired constructor( ) } Thread.sleep(1 * 1000) - val (status, errorMsg) = getTaskResult(userId, taskId).apply { + val (status, msg) = getTaskResult(userId, taskId, needProxy).apply { if (first == null) { return Pair(TaskStatusEnum.FAILED, second) } @@ -119,15 +125,23 @@ class KubernetesTaskClient @Autowired constructor( return when { status!!.isRunning() -> continue@loop status.isSuccess() -> { - Pair(TaskStatusEnum.SUCCEEDED, null) + Pair(TaskStatusEnum.SUCCEEDED, msg) } - else -> Pair(status, errorMsg) + else -> Pair(status, msg) } } } - private fun getTaskResult(userId: String, taskId: String): Pair { - val taskResponse = getTasksStatus(userId, taskId) + private fun getTaskResult( + userId: String, + taskId: String, + needProxy: Boolean + ): Pair { + val taskResponse = getTasksStatus( + userId = userId, + taskId = taskId, + needProxy = needProxy + ) val status = TaskStatusEnum.realNameOf(taskResponse.data?.status) if (taskResponse.isNotOk() || taskResponse.data == null) { // 创建失败 @@ -140,6 +154,6 @@ class KubernetesTaskClient @Autowired constructor( return Pair(status, taskResponse.data.detail) } - return Pair(status, null) + return Pair(status, taskResponse.data.detail) } } diff --git a/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/client/SecretClient.kt b/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/client/SecretClient.kt new file mode 100644 index 00000000000..17780a2c779 --- /dev/null +++ b/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/client/SecretClient.kt @@ -0,0 +1,170 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.dispatch.kubernetes.client + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue +import com.tencent.devops.common.api.util.JsonUtil +import com.tencent.devops.common.api.util.OkhttpUtils +import com.tencent.devops.common.dispatch.sdk.BuildFailureException +import com.tencent.devops.dispatch.kubernetes.pojo.KubernetesResult +import com.tencent.devops.dispatch.kubernetes.pojo.base.KubernetesRepo +import com.tencent.devops.dispatch.kubernetes.pojo.common.ErrorCodeEnum +import io.fabric8.kubernetes.api.model.Secret +import io.fabric8.kubernetes.api.model.SecretBuilder +import okhttp3.MediaType.Companion.toMediaTypeOrNull +import okhttp3.RequestBody +import org.apache.commons.codec.binary.Base64 +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired + +class SecretClient @Autowired constructor( + private val objectMapper: ObjectMapper, + private val clientCommon: KubernetesClientCommon +) { + + companion object { + private val logger = LoggerFactory.getLogger(SecretClient::class.java) + } + + fun createSecret( + userId: String, + namespace: String, + secret: Secret + ): KubernetesResult { + val url = "/api/namespace/$namespace/secrets" + val body = JsonUtil.toJson(secret) + logger.info("Create secret request url: $url, body: $body") + val request = clientCommon.microBaseRequest(url).post( + RequestBody.create( + "application/json; charset=utf-8".toMediaTypeOrNull(), + body + ) + ).build() + val responseBody = OkhttpUtils.doHttp(request).body!!.string() + logger.info("Create secret response: ${JsonUtil.toJson(responseBody)}") + return JsonUtil.getObjectMapper().readValue(responseBody) + } + + fun getSecretByName( + userId: String, + namespace: String, + secretName: String + ): KubernetesResult { + val url = "/api/namespace/$namespace/secrets/$secretName" + val request = clientCommon.microBaseRequest(url).get().build() + logger.info("Get secret: $secretName request url: $url, userId: $userId") + OkhttpUtils.doHttp(request).use { response -> + val responseContent = response.body!!.string() + logger.info("Get secret: $secretName response: $responseContent") + if (!response.isSuccessful) { + throw BuildFailureException( + errorType = ErrorCodeEnum.BCS_SYSTEM_ERROR.errorType, + errorCode = ErrorCodeEnum.BCS_SYSTEM_ERROR.errorCode, + formatErrorMessage = ErrorCodeEnum.BCS_SYSTEM_ERROR.getErrorMessage(), + errorMessage = "Fail to get secret,http response code: ${response.code}" + ) + } + return objectMapper.readValue(responseContent) + } + } + + fun deleteSecretByName( + userId: String, + namespace: String, + secretName: String + ): KubernetesResult { + val url = "/api/namespace/$namespace/secrets/$secretName" + val request = clientCommon.microBaseRequest(url).delete().build() + logger.info("Delete secret: $secretName request url: $url, userId: $userId") + OkhttpUtils.doHttp(request).use { response -> + val responseContent = response.body!!.string() + logger.info("Delete secret: $secretName response: $responseContent") + if (!response.isSuccessful) { + throw BuildFailureException( + errorType = ErrorCodeEnum.BCS_SYSTEM_ERROR.errorType, + errorCode = ErrorCodeEnum.BCS_SYSTEM_ERROR.errorCode, + formatErrorMessage = ErrorCodeEnum.BCS_SYSTEM_ERROR.getErrorMessage(), + errorMessage = "Fail to delete secret,http response code: ${response.code}" + ) + } + return objectMapper.readValue(responseContent) + } + } + + /** + * 创建k8s拉取镜像secret + * @param namespaceName 命名空间名称 + * @param secretName 秘钥名称 + * @param kubernetesRepoInfo k8s仓库信息 + */ + fun createImagePullSecret( + userId: String, + secretName: String, + namespaceName: String, + kubernetesRepoInfo: KubernetesRepo + ) { + var secret = getSecretByName(userId, namespaceName, secretName).data + logger.info("the secret is: $secret") + if (secret == null) { + val secretData: HashMap = HashMap(1) + val basicAuth = String( + Base64.encodeBase64("${kubernetesRepoInfo.username}:${kubernetesRepoInfo.password}".toByteArray()) + ) + var dockerCfg = String.format( + "{ " + + " \"auths\": { " + + " \"%s\": { " + + " \"username\": \"%s\", " + + " \"password\": \"%s\", " + + " \"email\": \"%s\", " + + " \"auth\": \"%s\" " + + " } " + + " } " + + "}", + kubernetesRepoInfo.registryUrl, + kubernetesRepoInfo.username, + kubernetesRepoInfo.password, + kubernetesRepoInfo.email, + basicAuth + ) + dockerCfg = String(Base64.encodeBase64(dockerCfg.toByteArray(Charsets.UTF_8)), Charsets.UTF_8) + secretData[".dockerconfigjson"] = dockerCfg + secret = SecretBuilder() + .withNewMetadata() + .withName(secretName) + .withNamespace(namespaceName) + .endMetadata() + .addToData(secretData) + .withType("kubernetes.io/dockerconfigjson") + .build() + createSecret(userId, namespaceName, secret) + logger.info("create new secret: $secret") + } + } +} diff --git a/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/client/ServiceClient.kt b/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/client/ServiceClient.kt new file mode 100644 index 00000000000..826da1da8a8 --- /dev/null +++ b/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/client/ServiceClient.kt @@ -0,0 +1,116 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.dispatch.kubernetes.client + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue +import com.tencent.devops.common.api.util.JsonUtil +import com.tencent.devops.common.api.util.OkhttpUtils +import com.tencent.devops.common.dispatch.sdk.BuildFailureException +import com.tencent.devops.dispatch.kubernetes.pojo.KubernetesResult +import com.tencent.devops.dispatch.kubernetes.pojo.common.ErrorCodeEnum +import io.fabric8.kubernetes.api.model.Service +import okhttp3.MediaType.Companion.toMediaTypeOrNull +import okhttp3.RequestBody +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired + +class ServiceClient @Autowired constructor( + private val objectMapper: ObjectMapper, + private val clientCommon: KubernetesClientCommon +) { + + companion object { + private val logger = LoggerFactory.getLogger(ServiceClient::class.java) + } + + fun createService( + userId: String, + namespace: String, + service: Service + ): KubernetesResult { + val url = "/api/namespace/$namespace/services" + val body = JsonUtil.toJson(service) + logger.info("$userId Create service request url: $url, body: $body") + val request = clientCommon.microBaseRequest(url).post( + RequestBody.create( + "application/json; charset=utf-8".toMediaTypeOrNull(), + body + ) + ).build() + val responseBody = OkhttpUtils.doHttp(request).body!!.string() + logger.info("Create service response: ${JsonUtil.toJson(responseBody)}") + return JsonUtil.getObjectMapper().readValue(responseBody) + } + + fun getServiceByName( + userId: String, + namespace: String, + serviceName: String + ): KubernetesResult { + val url = "/api/namespace/$namespace/services/$serviceName" + val request = clientCommon.microBaseRequest(url).get().build() + logger.info("Get service: $serviceName request url: $url, userId: $userId") + OkhttpUtils.doHttp(request).use { response -> + val responseContent = response.body!!.string() + logger.info("Get service: $serviceName response: $responseContent") + if (!response.isSuccessful) { + throw BuildFailureException( + errorType = ErrorCodeEnum.BCS_SYSTEM_ERROR.errorType, + errorCode = ErrorCodeEnum.BCS_SYSTEM_ERROR.errorCode, + formatErrorMessage = ErrorCodeEnum.BCS_SYSTEM_ERROR.getErrorMessage(), + errorMessage = "Fail to get service,http response code: ${response.code}" + ) + } + return objectMapper.readValue(responseContent) + } + } + + fun deleteServiceByName( + userId: String, + namespace: String, + serviceName: String + ): KubernetesResult { + val url = "/api/namespace/$namespace/services/$serviceName" + val request = clientCommon.microBaseRequest(url).delete().build() + logger.info("Delete service: $serviceName request url: $url, userId: $userId") + OkhttpUtils.doHttp(request).use { response -> + val responseContent = response.body!!.string() + logger.info("Delete service: $serviceName response: $responseContent") + if (!response.isSuccessful) { + throw BuildFailureException( + errorType = ErrorCodeEnum.BCS_SYSTEM_ERROR.errorType, + errorCode = ErrorCodeEnum.BCS_SYSTEM_ERROR.errorCode, + formatErrorMessage = ErrorCodeEnum.BCS_SYSTEM_ERROR.getErrorMessage(), + errorMessage = "Fail to delete service,http response code: ${response.code}" + ) + } + return objectMapper.readValue(responseContent) + } + } +} diff --git a/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/config/KubernetesBeanConfiguration.kt b/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/config/KubernetesBeanConfiguration.kt index 9d28886d8d8..fed7be9143e 100644 --- a/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/config/KubernetesBeanConfiguration.kt +++ b/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/config/KubernetesBeanConfiguration.kt @@ -35,10 +35,14 @@ import com.tencent.devops.dispatch.kubernetes.bcs.client.BcsJobClient import com.tencent.devops.dispatch.kubernetes.bcs.client.BcsTaskClient import com.tencent.devops.dispatch.kubernetes.bcs.service.BcsContainerService import com.tencent.devops.dispatch.kubernetes.bcs.service.BcsJobService +import com.tencent.devops.dispatch.kubernetes.client.DeploymentClient +import com.tencent.devops.dispatch.kubernetes.client.IngressClient import com.tencent.devops.dispatch.kubernetes.client.KubernetesBuilderClient import com.tencent.devops.dispatch.kubernetes.client.KubernetesClientCommon import com.tencent.devops.dispatch.kubernetes.client.KubernetesJobClient import com.tencent.devops.dispatch.kubernetes.client.KubernetesTaskClient +import com.tencent.devops.dispatch.kubernetes.client.SecretClient +import com.tencent.devops.dispatch.kubernetes.client.ServiceClient import com.tencent.devops.dispatch.kubernetes.components.LogsPrinter import com.tencent.devops.dispatch.kubernetes.interfaces.CommonService import com.tencent.devops.dispatch.kubernetes.service.CoreCommonService @@ -165,4 +169,36 @@ class KubernetesBeanConfiguration { fun bcsJobService( bcsJobClient: BcsJobClient ) = BcsJobService(bcsJobClient) + + @Bean + @Primary + @ConditionalOnProperty(prefix = "kubernetes", name = ["enable"], havingValue = "true") + fun deploymentClient( + objectMapper: ObjectMapper, + clientCommon: KubernetesClientCommon + ) = DeploymentClient(objectMapper, clientCommon) + + @Bean + @Primary + @ConditionalOnProperty(prefix = "kubernetes", name = ["enable"], havingValue = "true") + fun ingressClient( + objectMapper: ObjectMapper, + clientCommon: KubernetesClientCommon + ) = IngressClient(objectMapper, clientCommon) + + @Bean + @Primary + @ConditionalOnProperty(prefix = "kubernetes", name = ["enable"], havingValue = "true") + fun secretClient( + objectMapper: ObjectMapper, + clientCommon: KubernetesClientCommon + ) = SecretClient(objectMapper, clientCommon) + + @Bean + @Primary + @ConditionalOnProperty(prefix = "kubernetes", name = ["enable"], havingValue = "true") + fun serviceClient( + objectMapper: ObjectMapper, + clientCommon: KubernetesClientCommon + ) = ServiceClient(objectMapper, clientCommon) } diff --git a/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/pojo/InspectImageReq.kt b/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/pojo/InspectImageReq.kt new file mode 100644 index 00000000000..ad414a3d96c --- /dev/null +++ b/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/pojo/InspectImageReq.kt @@ -0,0 +1,12 @@ +package com.tencent.devops.dispatch.kubernetes.pojo + +data class InspectImageReq( + val name: String, + val ref: String, + val cred: InspectImageCredential +) + +data class InspectImageCredential( + val username: String, + val password: String +) diff --git a/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/service/KubernetesContainerService.kt b/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/service/KubernetesContainerService.kt index 31e06fe0769..ed74416a5bc 100644 --- a/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/service/KubernetesContainerService.kt +++ b/src/backend/ci/core/dispatch/biz-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/service/KubernetesContainerService.kt @@ -53,6 +53,7 @@ import com.tencent.devops.dispatch.kubernetes.pojo.BK_START_BUILD_CONTAINER_FAIL import com.tencent.devops.dispatch.kubernetes.pojo.BuildAndPushImage import com.tencent.devops.dispatch.kubernetes.pojo.BuildAndPushImageInfo import com.tencent.devops.dispatch.kubernetes.pojo.Builder +import com.tencent.devops.dispatch.kubernetes.pojo.Credential import com.tencent.devops.dispatch.kubernetes.pojo.DeleteBuilderParams import com.tencent.devops.dispatch.kubernetes.pojo.DispatchBuildLog import com.tencent.devops.dispatch.kubernetes.pojo.KubernetesBuilderStatusEnum @@ -298,10 +299,10 @@ class KubernetesContainerService @Autowired constructor( } } - override fun waitTaskFinish(userId: String, taskId: String): DispatchBuildTaskStatus { - val startResult = kubernetesTaskClient.waitTaskFinish(userId, taskId) + override fun waitTaskFinish(userId: String, taskId: String, needProxy: Boolean): DispatchBuildTaskStatus { + val startResult = kubernetesTaskClient.waitTaskFinish(userId, taskId, needProxy) return if (startResult.first == TaskStatusEnum.SUCCEEDED) { - DispatchBuildTaskStatus(DispatchBuildTaskStatusEnum.SUCCEEDED, null) + DispatchBuildTaskStatus(DispatchBuildTaskStatusEnum.SUCCEEDED, startResult.second) } else { DispatchBuildTaskStatus(DispatchBuildTaskStatusEnum.FAILED, startResult.second) } @@ -425,6 +426,17 @@ class KubernetesContainerService @Autowired constructor( return DispatchTaskResp(kubernetesJobClient.buildAndPushImage(userId, info)) } + override fun inspectDockerImage(userId: String, pool: Pool): String { + return kubernetesBuilderClient.inspectDockerImage( + userId = userId, + imageName = pool.container ?: "", + credential = pool.credential ?: Credential( + user = "", + password = "" + ) + ) + } + private fun getBuilderName(): String { return "build${System.currentTimeMillis()}-" + RandomStringUtils.randomAlphabetic(8).lowercase(Locale.getDefault()) diff --git a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/ThirdPartyDispatchService.kt b/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/ThirdPartyDispatchService.kt index 8bfafaf6c9c..5fa689af8ee 100644 --- a/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/ThirdPartyDispatchService.kt +++ b/src/backend/ci/core/dispatch/biz-dispatch/src/main/kotlin/com/tencent/devops/dispatch/service/ThirdPartyDispatchService.kt @@ -45,6 +45,8 @@ import com.tencent.devops.common.pipeline.type.agent.ThirdPartyAgentIDDispatchTy import com.tencent.devops.common.pipeline.type.agent.ThirdPartyDevCloudDispatchType import com.tencent.devops.common.redis.RedisLockByValue import com.tencent.devops.common.redis.RedisOperation +import com.tencent.devops.common.service.config.CommonConfig +import com.tencent.devops.common.service.utils.HomeHostUtil import com.tencent.devops.common.web.utils.I18nUtil import com.tencent.devops.dispatch.constants.AGENT_REUSE_MUTEX_REDISPATCH import com.tencent.devops.dispatch.constants.AGENT_REUSE_MUTEX_WAIT_REUSED_ENV @@ -87,13 +89,14 @@ class ThirdPartyDispatchService @Autowired constructor( private val client: Client, private val redisOperation: RedisOperation, private val buildLogPrinter: BuildLogPrinter, + private val commonConfig: CommonConfig, private val thirdPartyAgentBuildRedisUtils: ThirdPartyAgentBuildRedisUtils, private val thirdPartyAgentBuildService: ThirdPartyAgentService ) { fun canDispatch(event: PipelineAgentStartupEvent) = event.dispatchType is ThirdPartyAgentIDDispatchType || - event.dispatchType is ThirdPartyAgentEnvDispatchType || - event.dispatchType is ThirdPartyDevCloudDispatchType + event.dispatchType is ThirdPartyAgentEnvDispatchType || + event.dispatchType is ThirdPartyDevCloudDispatchType fun startUp(dispatchMessage: DispatchMessage) { when (dispatchMessage.event.dispatchType) { @@ -150,11 +153,11 @@ class ThirdPartyDispatchService @Autowired constructor( throw DispatchRetryMQException( errorCodeEnum = ErrorCodeEnum.BUILD_ENV_PREPARATION, errorMessage = "${dispatchMessage.event.buildId}|${dispatchMessage.event.vmSeqId} " + - I18nUtil.getCodeLanMessage( - messageCode = BK_QUEUE_TIMEOUT_MINUTES, - language = I18nUtil.getDefaultLocaleLanguage(), - params = arrayOf("${dispatchMessage.event.queueTimeoutMinutes}") - ) + I18nUtil.getCodeLanMessage( + messageCode = BK_QUEUE_TIMEOUT_MINUTES, + language = I18nUtil.getDefaultLocaleLanguage(), + params = arrayOf("${dispatchMessage.event.queueTimeoutMinutes}") + ) ) } @@ -174,7 +177,7 @@ class ThirdPartyDispatchService @Autowired constructor( ThirdPartyAgentIDDispatchType( displayName = agentId, workspace = dispatchType.workspace, - agentType = AgentType.ID, + agentType = AgentType.REUSE_JOB_ID, dockerInfo = dispatchType.dockerInfo, reusedInfo = dispatchType.reusedInfo ) @@ -206,7 +209,6 @@ class ThirdPartyDispatchService @Autowired constructor( dispatchMessage: DispatchMessage, dispatchType: ThirdPartyAgentIDDispatchType ) { - val agentResult = if (dispatchType.idType()) { client.get(ServiceThirdPartyAgentResource::class) .getAgentById(dispatchMessage.event.projectId, dispatchType.displayName) @@ -221,7 +223,7 @@ class ThirdPartyDispatchService @Autowired constructor( errorCode = ErrorCodeEnum.GET_BUILD_AGENT_ERROR.errorCode, formatErrorMessage = ErrorCodeEnum.GET_BUILD_AGENT_ERROR.formatErrorMessage, errorMessage = ErrorCodeEnum.GET_BUILD_AGENT_ERROR.getErrorMessage() + - "(System Error) - ${agentResult.message}" + "(System Error) - ${agentResult.message}" ) } @@ -231,7 +233,7 @@ class ThirdPartyDispatchService @Autowired constructor( errorCode = ErrorCodeEnum.VM_STATUS_ERROR.errorCode, formatErrorMessage = ErrorCodeEnum.VM_STATUS_ERROR.formatErrorMessage, errorMessage = ErrorCodeEnum.VM_STATUS_ERROR.getErrorMessage() + - "- ${dispatchType.displayName}| status: (${agentResult.agentStatus?.name})" + "- ${dispatchType.displayName}| status: (${agentResult.agentStatus?.name})" ) } @@ -241,7 +243,7 @@ class ThirdPartyDispatchService @Autowired constructor( errorCode = ErrorCodeEnum.FOUND_AGENT_ERROR.errorCode, formatErrorMessage = ErrorCodeEnum.FOUND_AGENT_ERROR.formatErrorMessage, errorMessage = ErrorCodeEnum.FOUND_AGENT_ERROR.getErrorMessage() + - "(System Error) - $dispatchType agent is null" + "(System Error) - $dispatchType agent is null" ) } @@ -303,8 +305,8 @@ class ThirdPartyDispatchService @Autowired constructor( } // #10082 对于复用的机器和被复用的，需要加锁校验看看这台机器能不能使用 + val lockKey = AgentReuseMutex.genAgentReuseMutexLockKey(event.projectId, agent.agentId) if (hasReuseMutex) { - val lockKey = AgentReuseMutex.genAgentReuseMutexLockKey(event.projectId, agent.agentId) val lock = RedisLockByValue( redisOperation = redisOperation, lockKey = lockKey, @@ -313,16 +315,7 @@ class ThirdPartyDispatchService @Autowired constructor( ) // 没有拿到锁说明现在这台机被复用互斥占用不能选 if (!lock.tryLock()) { - log( - dispatchMessage.event, - I18nUtil.getCodeLanMessage( - messageCode = AGENT_REUSE_MUTEX_REDISPATCH, - language = I18nUtil.getDefaultLocaleLanguage(), - params = arrayOf( - "${agent.agentId}|${agent.hostname}/${agent.ip}", redisOperation.get(lockKey) ?: "" - ) - ) - ) + logAgentReuse(lockKey, dispatchMessage, agent) return false } try { @@ -348,6 +341,19 @@ class ThirdPartyDispatchService @Autowired constructor( } catch (e: Exception) { logger.error("inQueue|doAgentInQueue|error", e) } + } else if (redisOperation.get(lockKey) != null) { + // 没有复用逻辑的需要检查下如果这个机器剩一个可调度空间且有复用锁那么不能进行调度 + val checkRes = if (dockerInfo != null) { + ((agent.dockerParallelTaskCount ?: 4) - + thirdPartyAgentBuildService.getDockerRunningBuilds(agent.agentId)) <= 1 + } else { + ((agent.parallelTaskCount ?: 4) - + thirdPartyAgentBuildService.getRunningBuilds(agent.agentId)) <= 1 + } + if (checkRes) { + logAgentReuse(lockKey, dispatchMessage, agent) + return false + } } // #5806 入库失败就不再写Redis @@ -374,8 +380,7 @@ class ThirdPartyDispatchService @Autowired constructor( saveAgentInfoToBuildDetail(dispatchMessage = dispatchMessage, agent = agent) logger.info( - "${event.buildId}|START_AGENT_BY_ID|j(${event.vmSeqId})|" + - "agent=${agent.agentId}" + "${event.buildId}|START_AGENT_BY_ID|j(${event.vmSeqId})|agent=${agent.agentId}" ) log( dispatchMessage.event, @@ -400,6 +405,52 @@ class ThirdPartyDispatchService @Autowired constructor( } } + private fun logAgentReuse( + lockKey: String, + dispatchMessage: DispatchMessage, + agent: ThirdPartyAgent + ) { + val lockedBuildId = redisOperation.get(lockKey) + if (lockedBuildId.isNullOrBlank()) { + log( + dispatchMessage.event, + I18nUtil.getCodeLanMessage( + messageCode = AGENT_REUSE_MUTEX_REDISPATCH, + language = I18nUtil.getDefaultLocaleLanguage(), + params = arrayOf( + "${agent.agentId}|${agent.hostname}/${agent.ip}", lockedBuildId ?: "" + ) + ) + ) + return + } + val msg = redisOperation.get(AgentReuseMutex.genAgentReuseMutexLinkTipKey(lockedBuildId))?.let { s -> + val endIndex = s.indexOf("_") + val pipelineId = s.substring(0, endIndex) + val linkTip = s.substring(endIndex + 1) + val link = "${ + HomeHostUtil.getHost( + commonConfig.devopsHostGateway!! + ) + }/console/pipeline/${dispatchMessage.event.projectId}/$pipelineId/detail/$lockedBuildId" + if (lockedBuildId != dispatchMessage.event.buildId) { + "$linkTip$lockedBuildId" + } else { + linkTip + } + } ?: "" + logWarn( + dispatchMessage.event, + I18nUtil.getCodeLanMessage( + messageCode = AGENT_REUSE_MUTEX_REDISPATCH, + language = I18nUtil.getDefaultLocaleLanguage(), + params = arrayOf( + "${agent.agentId}|${agent.hostname}/${agent.ip}", lockedBuildId ?: "" + ) + ) + msg + ) + } + private fun inQueue( agent: ThirdPartyAgent, dispatchMessage: DispatchMessage, @@ -509,7 +560,7 @@ class ThirdPartyDispatchService @Autowired constructor( if (agentsResult.status == Response.Status.FORBIDDEN.statusCode) { logger.warn( "${dispatchMessage.event.buildId}|START_AGENT_FAILED_FORBIDDEN|" + - "j(${dispatchMessage.event.vmSeqId})|dispatchType=$dispatchType|err=${agentsResult.message}" + "j(${dispatchMessage.event.vmSeqId})|dispatchType=$dispatchType|err=${agentsResult.message}" ) throw BuildFailureException( @@ -523,7 +574,7 @@ class ThirdPartyDispatchService @Autowired constructor( if (agentsResult.isNotOk()) { logger.warn( "${dispatchMessage.event.buildId}|START_AGENT_FAILED|" + - "j(${dispatchMessage.event.vmSeqId})|dispatchType=$dispatchType|err=${agentsResult.message}" + "j(${dispatchMessage.event.vmSeqId})|dispatchType=$dispatchType|err=${agentsResult.message}" ) logDebug( @@ -537,14 +588,14 @@ class ThirdPartyDispatchService @Autowired constructor( throw DispatchRetryMQException( errorCodeEnum = ErrorCodeEnum.FOUND_AGENT_ERROR, errorMessage = ErrorCodeEnum.GET_BUILD_AGENT_ERROR.getErrorMessage() + - "(System Error) - ${dispatchType.envName}: ${agentsResult.message}" + "(System Error) - ${dispatchType.envName}: ${agentsResult.message}" ) } if (agentsResult.data == null) { logger.warn( "${dispatchMessage.event.buildId}|START_AGENT_FAILED|j(${dispatchMessage.event.vmSeqId})|" + - "dispatchType=$dispatchType|err=null agents" + "dispatchType=$dispatchType|err=null agents" ) logDebug( dispatchMessage.event, @@ -557,7 +608,7 @@ class ThirdPartyDispatchService @Autowired constructor( throw DispatchRetryMQException( errorCodeEnum = ErrorCodeEnum.FOUND_AGENT_ERROR, errorMessage = ErrorCodeEnum.GET_BUILD_AGENT_ERROR.getErrorMessage() + - "System Error) - ${dispatchType.envName}: agent is null" + "System Error) - ${dispatchType.envName}: agent is null" ) } @@ -573,7 +624,7 @@ class ThirdPartyDispatchService @Autowired constructor( if (agentResData.isEmpty()) { logger.warn( "${dispatchMessage.event.buildId}|START_AGENT_FAILED|j(${dispatchMessage.event.vmSeqId})|" + - "dispatchType=$dispatchType|err=empty agents" + "dispatchType=$dispatchType|err=empty agents" ) throw DispatchRetryMQException( errorCodeEnum = ErrorCodeEnum.VM_NODE_NULL, @@ -725,7 +776,7 @@ class ThirdPartyDispatchService @Autowired constructor( logger.warn( "buildByEnvId|{} has singleNodeConcurrency {} but env {}|job {} null", dispatchMessage.event.buildId, - dispatchMessage.event.allNodeConcurrency, + dispatchMessage.event.singleNodeConcurrency, envId, dispatchMessage.event.jobId ) @@ -863,15 +914,15 @@ class ThirdPartyDispatchService @Autowired constructor( logDebug( dispatchMessage.event, "retry: ${dispatchMessage.event.retryTime} | " + - I18nUtil.getCodeLanMessage( - messageCode = BK_SEARCHING_AGENT, - language = I18nUtil.getDefaultLocaleLanguage() - ) + I18nUtil.getCodeLanMessage( + messageCode = BK_SEARCHING_AGENT, + language = I18nUtil.getDefaultLocaleLanguage() + ) ) if (startEmptyAgents(dispatchMessage, dispatchType, pbAgents, hasTryAgents, envId)) { logger.info( "${dispatchMessage.event.buildId}|START_AGENT|j(${dispatchMessage.event.vmSeqId})|" + - "dispatchType=$dispatchType|Get Lv.1" + "dispatchType=$dispatchType|Get Lv.1" ) return true } @@ -879,10 +930,10 @@ class ThirdPartyDispatchService @Autowired constructor( logDebug( dispatchMessage.event, "retry: ${dispatchMessage.event.retryTime} | " + - I18nUtil.getCodeLanMessage( - messageCode = BK_MAX_BUILD_SEARCHING_AGENT, - language = I18nUtil.getDefaultLocaleLanguage() - ) + I18nUtil.getCodeLanMessage( + messageCode = BK_MAX_BUILD_SEARCHING_AGENT, + language = I18nUtil.getDefaultLocaleLanguage() + ) ) /** * 次高优先级的agent: 最近构建机中使用过这个构建机,并且当前有构建任务,选当前正在运行任务最少的构建机(没有达到当前构建机的最大并发数) @@ -890,7 +941,7 @@ class ThirdPartyDispatchService @Autowired constructor( if (startAvailableAgents(dispatchMessage, dispatchType, pbAgents, hasTryAgents, envId)) { logger.info( "${dispatchMessage.event.buildId}|START_AGENT|j(${dispatchMessage.event.vmSeqId})|" + - "dispatchType=$dispatchType|Get Lv.2" + "dispatchType=$dispatchType|Get Lv.2" ) return true } @@ -898,10 +949,10 @@ class ThirdPartyDispatchService @Autowired constructor( logDebug( dispatchMessage.event, "retry: ${dispatchMessage.event.retryTime} | " + - I18nUtil.getCodeLanMessage( - messageCode = BK_SEARCHING_AGENT_MOST_IDLE, - language = I18nUtil.getDefaultLocaleLanguage() - ) + I18nUtil.getCodeLanMessage( + messageCode = BK_SEARCHING_AGENT_MOST_IDLE, + language = I18nUtil.getDefaultLocaleLanguage() + ) ) val allAgents = sortAgent( dispatchMessage = dispatchMessage, @@ -916,7 +967,7 @@ class ThirdPartyDispatchService @Autowired constructor( if (startEmptyAgents(dispatchMessage, dispatchType, allAgents, hasTryAgents, envId)) { logger.info( "${dispatchMessage.event.buildId}|START_AGENT|j(${dispatchMessage.event.vmSeqId})|" + - "dispatchType=$dispatchType|pickup Lv.3" + "dispatchType=$dispatchType|pickup Lv.3" ) return true } @@ -924,10 +975,10 @@ class ThirdPartyDispatchService @Autowired constructor( logDebug( dispatchMessage.event, "retry: ${dispatchMessage.event.retryTime} | " + - I18nUtil.getCodeLanMessage( - messageCode = BK_SEARCHING_AGENT_PARALLEL_AVAILABLE, - language = I18nUtil.getDefaultLocaleLanguage() - ) + I18nUtil.getCodeLanMessage( + messageCode = BK_SEARCHING_AGENT_PARALLEL_AVAILABLE, + language = I18nUtil.getDefaultLocaleLanguage() + ) ) /** * 第四优先级的agent: 当前有构建任务,选当前正在运行任务最少的构建机(没有达到当前构建机的最大并发数) @@ -936,7 +987,7 @@ class ThirdPartyDispatchService @Autowired constructor( ) { logger.info( "${dispatchMessage.event.buildId}|START_AGENT|j(${dispatchMessage.event.vmSeqId})|" + - "dispatchType=$dispatchType|Get Lv.4" + "dispatchType=$dispatchType|Get Lv.4" ) return true } @@ -999,7 +1050,7 @@ class ThirdPartyDispatchService @Autowired constructor( if (startEnvAgentBuild(dispatchMessage, agent, dispatchType, hasTryAgents, envId)) { logger.info( "[${dispatchMessage.event.projectId}|$[${dispatchMessage.event.pipelineId}|" + - "${dispatchMessage.event.buildId}|${agent.agentId}] Success to start the build" + "${dispatchMessage.event.buildId}|${agent.agentId}] Success to start the build" ) return true } @@ -1071,13 +1122,13 @@ class ThirdPartyDispatchService @Autowired constructor( dockerRunningCnt: Int ): Boolean { return if (dockerBuilder) { - agent.dockerParallelTaskCount != null && - agent.dockerParallelTaskCount!! > 0 && - agent.dockerParallelTaskCount!! > dockerRunningCnt + (agent.dockerParallelTaskCount == 0) || (agent.dockerParallelTaskCount != null && + agent.dockerParallelTaskCount!! > 0 && + agent.dockerParallelTaskCount!! > dockerRunningCnt) } else { - agent.parallelTaskCount != null && - agent.parallelTaskCount!! > 0 && - agent.parallelTaskCount!! > runningCnt + (agent.parallelTaskCount == 0) || (agent.parallelTaskCount != null && + agent.parallelTaskCount!! > 0 && + agent.parallelTaskCount!! > runningCnt) } } } diff --git a/src/backend/ci/core/dispatch/common-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/interfaces/ContainerService.kt b/src/backend/ci/core/dispatch/common-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/interfaces/ContainerService.kt index 6c93e6c9656..c407e2d01ff 100644 --- a/src/backend/ci/core/dispatch/common-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/interfaces/ContainerService.kt +++ b/src/backend/ci/core/dispatch/common-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/interfaces/ContainerService.kt @@ -117,7 +117,8 @@ interface ContainerService { */ fun waitTaskFinish( userId: String, - taskId: String + taskId: String, + needProxy: Boolean = true ): DispatchBuildTaskStatus /** @@ -157,4 +158,9 @@ interface ContainerService { buildId: String, dispatchBuildImageReq: DispatchBuildImageReq ): DispatchTaskResp + + /** + * inspect镜像接口 + */ + fun inspectDockerImage(userId: String, pool: Pool): String } diff --git a/src/backend/ci/core/dispatch/common-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/pojo/builds/DispatchBuildTaskStatus.kt b/src/backend/ci/core/dispatch/common-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/pojo/builds/DispatchBuildTaskStatus.kt index deab0613e92..d07422bc079 100644 --- a/src/backend/ci/core/dispatch/common-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/pojo/builds/DispatchBuildTaskStatus.kt +++ b/src/backend/ci/core/dispatch/common-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/pojo/builds/DispatchBuildTaskStatus.kt @@ -29,7 +29,7 @@ package com.tencent.devops.dispatch.kubernetes.pojo.builds data class DispatchBuildTaskStatus( val status: DispatchBuildTaskStatusEnum, - val errMsg: String? + val msg: String? ) enum class DispatchBuildTaskStatusEnum { diff --git a/src/backend/ci/core/dispatch/common-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/pojo/image/DockerImageInspection.kt b/src/backend/ci/core/dispatch/common-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/pojo/image/DockerImageInspection.kt new file mode 100644 index 00000000000..12da161343d --- /dev/null +++ b/src/backend/ci/core/dispatch/common-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/pojo/image/DockerImageInspection.kt @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.dispatch.kubernetes.pojo.image + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties + +@JsonIgnoreProperties(ignoreUnknown = true) +data class DockerImageInspection( + val arch: String? = "", + val author: String? = "", + val comment: String? = "", + val created: String? = "", + val dockerVersion: String? = "", + val id: String? = "", + val os: String? = "", + val osVersion: String? = "", + val parent: String? = "", + val size: Long? = 0, + val repoTags: List? = null, + val repoDigests: List? = null, + val virtualSize: Long? = 0 +) diff --git a/src/backend/ci/core/dispatch/common-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/pojo/image/InspectImageResp.kt b/src/backend/ci/core/dispatch/common-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/pojo/image/InspectImageResp.kt new file mode 100644 index 00000000000..85a569dc946 --- /dev/null +++ b/src/backend/ci/core/dispatch/common-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/pojo/image/InspectImageResp.kt @@ -0,0 +1,12 @@ +package com.tencent.devops.dispatch.kubernetes.pojo.image + +data class InspectImageResp( + val arch: String, // 架构 + val os: String, // 系统 + val size: Long, // 大小 + val created: String, // 创建时间 + val id: String, // HashId + val author: String, // 镜像作者 + val parent: String, // 父级镜像 + val osVersion: String // 系统版本 +) diff --git a/src/backend/ci/core/dispatch/common-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/resource/service/ServiceDockerImageResourceImpl.kt b/src/backend/ci/core/dispatch/common-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/resource/service/ServiceDockerImageResourceImpl.kt new file mode 100644 index 00000000000..e521edb22a6 --- /dev/null +++ b/src/backend/ci/core/dispatch/common-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/resource/service/ServiceDockerImageResourceImpl.kt @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.dispatch.kubernetes.resource.service + +import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.common.web.RestResource +import com.tencent.devops.dispatch.kubernetes.api.service.ServiceDockerImageResource +import com.tencent.devops.dispatch.kubernetes.service.DispatchBaseImageService +import com.tencent.devops.dispatch.kubernetes.pojo.CheckDockerImageRequest +import com.tencent.devops.dispatch.kubernetes.pojo.CheckDockerImageResponse +import org.springframework.beans.factory.annotation.Autowired + +@RestResource +class ServiceDockerImageResourceImpl @Autowired constructor( + private val dispatchBaseImageService: DispatchBaseImageService +) : ServiceDockerImageResource { + + override fun checkDockerImage( + userId: String, + checkDockerImageRequestList: List + ): Result> { + return Result(dispatchBaseImageService.checkDockerImage(userId, checkDockerImageRequestList)) + } +} diff --git a/src/backend/ci/core/dispatch/common-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/service/DispatchBaseDebugService.kt b/src/backend/ci/core/dispatch/common-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/service/DispatchBaseDebugService.kt index 93882de771b..371e473ee8c 100644 --- a/src/backend/ci/core/dispatch/common-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/service/DispatchBaseDebugService.kt +++ b/src/backend/ci/core/dispatch/common-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/service/DispatchBaseDebugService.kt @@ -257,7 +257,7 @@ class DispatchBaseDebugService @Autowired constructor( logger.info("stop debug $debugBuilderName success.") } else { // 停不掉，尝试删除 - logger.info("stop debug $debugBuilderName failed, msg: ${opResult.errMsg}") + logger.info("stop debug $debugBuilderName failed, msg: ${opResult.msg}") logger.info("stop debug $debugBuilderName failed, try to delete it.") containerServiceFactory.load(projectId).operateBuilder( buildId = "", @@ -318,10 +318,10 @@ class DispatchBaseDebugService @Autowired constructor( // 启动成功 logger.info("$userId start ${dockerRoutingType.name} builder success") } else { - logger.error("$userId start ${dockerRoutingType.name} builder failed, msg: ${startResult.errMsg}") + logger.error("$userId start ${dockerRoutingType.name} builder failed, msg: ${startResult.msg}") throw ErrorCodeException( errorCode = BK_BUILD_MACHINE_STARTUP_FAILED, - params = arrayOf(startResult.errMsg ?: "") + params = arrayOf(startResult.msg ?: "") ) } } diff --git a/src/backend/ci/core/dispatch/common-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/service/DispatchBaseImageService.kt b/src/backend/ci/core/dispatch/common-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/service/DispatchBaseImageService.kt new file mode 100644 index 00000000000..261fd9c6cef --- /dev/null +++ b/src/backend/ci/core/dispatch/common-dispatch-kubernetes/src/main/kotlin/com/tencent/devops/dispatch/kubernetes/service/DispatchBaseImageService.kt @@ -0,0 +1,109 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.dispatch.kubernetes.service + +import com.tencent.devops.common.api.util.JsonUtil +import com.tencent.devops.dispatch.kubernetes.pojo.CheckDockerImageRequest +import com.tencent.devops.dispatch.kubernetes.pojo.CheckDockerImageResponse +import com.tencent.devops.dispatch.kubernetes.pojo.Credential +import com.tencent.devops.dispatch.kubernetes.pojo.Pool +import com.tencent.devops.dispatch.kubernetes.pojo.builds.DispatchBuildTaskStatusEnum +import com.tencent.devops.dispatch.kubernetes.pojo.image.InspectImageResp +import com.tencent.devops.dispatch.kubernetes.service.factory.ContainerServiceFactory +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service + +@Service +class DispatchBaseImageService @Autowired constructor( + private val containerServiceFactory: ContainerServiceFactory +) { + + companion object { + private val logger = LoggerFactory.getLogger(DispatchBaseImageService::class.java) + } + + fun checkDockerImage( + userId: String, + checkDockerImageRequestList: List + ): List { + val imageInspectList = mutableListOf() + checkDockerImageRequestList.parallelStream().forEach { + // 拉镜像 + val taskId = containerServiceFactory.load("").inspectDockerImage( + userId = userId, + pool = Pool( + container = it.imageName, + credential = Credential( + user = it.registryUser ?: "", + password = it.registryPwd ?: "" + ) + ) + ) + + val taskResult = containerServiceFactory.load("").waitTaskFinish(userId, taskId, false) + if (taskResult.status == DispatchBuildTaskStatusEnum.SUCCEEDED) { + logger.info("CheckDockerImage $userId pull ${it.imageName} success.") + val inspectImageResp = JsonUtil.to(taskResult.msg ?: "", InspectImageResp::class.java) + imageInspectList.add( + CheckDockerImageResponse( + errorCode = 0, + errorMessage = "", + arch = inspectImageResp.arch, + author = inspectImageResp.author, + created = inspectImageResp.created, + id = inspectImageResp.id, + os = inspectImageResp.os, + osVersion = inspectImageResp.osVersion, + parent = inspectImageResp.parent, + size = inspectImageResp.size + ) + ) + } else { + imageInspectList.add( + CheckDockerImageResponse( + errorCode = -1, + errorMessage = taskResult.msg, + arch = "", + author = "", + created = "", + id = "", + os = "", + osVersion = "", + parent = "", + size = 0 + ) + ) + + return@forEach + } + } + + return imageInspectList + } +} diff --git a/src/backend/ci/core/dockerhost/biz-dockerhost/src/main/kotlin/com/tencent/devops/dockerhost/services/DockerHostBuildService.kt b/src/backend/ci/core/dockerhost/biz-dockerhost/src/main/kotlin/com/tencent/devops/dockerhost/services/DockerHostBuildService.kt index 3c739cd6087..05264a0e406 100644 --- a/src/backend/ci/core/dockerhost/biz-dockerhost/src/main/kotlin/com/tencent/devops/dockerhost/services/DockerHostBuildService.kt +++ b/src/backend/ci/core/dockerhost/biz-dockerhost/src/main/kotlin/com/tencent/devops/dockerhost/services/DockerHostBuildService.kt @@ -754,7 +754,7 @@ class DockerHostBuildService( val date = sdf.parse(utcTimeLocal) val startTimestamp = date.time val nowTimestamp = System.currentTimeMillis() - return (nowTimestamp - startTimestamp) > (8 * 3600 * 1000) + return (nowTimestamp - startTimestamp) > (16 * 3600 * 1000) } return false diff --git a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/api/thirdpartyagent/ExternalThirdPartyAgentResource.kt b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/api/thirdpartyagent/ExternalThirdPartyAgentResource.kt index cfc4a394498..06b27842e4e 100644 --- a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/api/thirdpartyagent/ExternalThirdPartyAgentResource.kt +++ b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/api/thirdpartyagent/ExternalThirdPartyAgentResource.kt @@ -27,12 +27,15 @@ package com.tencent.devops.environment.api.thirdpartyagent +import com.tencent.devops.common.api.pojo.OS import com.tencent.devops.common.web.annotation.BkField -import io.swagger.v3.oas.annotations.tags.Tag +import com.tencent.devops.environment.constant.BATCH_TOKEN_HEADER import io.swagger.v3.oas.annotations.Operation import io.swagger.v3.oas.annotations.Parameter +import io.swagger.v3.oas.annotations.tags.Tag import javax.ws.rs.Consumes import javax.ws.rs.GET +import javax.ws.rs.HeaderParam import javax.ws.rs.Path import javax.ws.rs.PathParam import javax.ws.rs.Produces @@ -101,4 +104,20 @@ interface ExternalThirdPartyAgentResource { @BkField(minLength = 3, maxLength = 32) agentHashId: String ): Response + + @Operation(summary = "下载agent批量安装脚本") + @GET + @Path("/{os}/batchInstall") + @Produces(MediaType.APPLICATION_OCTET_STREAM) + fun batchDownloadAgentInstallScript( + @Parameter(description = "TOKEN", required = false) + @HeaderParam(BATCH_TOKEN_HEADER) + token: String, + @Parameter(description = "操作系统", required = true) + @PathParam("os") + os: OS, + @Parameter(description = "网关地域", required = false) + @QueryParam("zoneName") + zoneName: String? + ): Response } diff --git a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/api/thirdpartyagent/ServiceThirdPartyAgentResource.kt b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/api/thirdpartyagent/ServiceThirdPartyAgentResource.kt index 0fd64c4f31a..afe18f996ba 100644 --- a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/api/thirdpartyagent/ServiceThirdPartyAgentResource.kt +++ b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/api/thirdpartyagent/ServiceThirdPartyAgentResource.kt @@ -40,10 +40,13 @@ import com.tencent.devops.common.api.pojo.agent.NewHeartbeatInfo import com.tencent.devops.common.api.pojo.agent.UpgradeItem import com.tencent.devops.common.web.annotation.BkField import com.tencent.devops.environment.pojo.AgentPipelineRefRequest +import com.tencent.devops.environment.pojo.EnvVar import com.tencent.devops.environment.pojo.slave.SlaveGateway import com.tencent.devops.environment.pojo.thirdpartyagent.AgentBuildDetail import com.tencent.devops.environment.pojo.thirdpartyagent.AgentPipelineRef import com.tencent.devops.environment.pojo.thirdpartyagent.AskHeartbeatResponse +import com.tencent.devops.environment.pojo.thirdpartyagent.BatchFetchAgentData +import com.tencent.devops.environment.pojo.thirdpartyagent.BatchUpdateAgentEnvVar import com.tencent.devops.environment.pojo.thirdpartyagent.EnvNodeAgent import com.tencent.devops.environment.pojo.thirdpartyagent.ThirdPartyAgent import com.tencent.devops.environment.pojo.thirdpartyagent.ThirdPartyAgentDetail @@ -416,4 +419,32 @@ interface ServiceThirdPartyAgentResource { @PathParam("envName") envName: String ): Result>> + + @Operation(summary = "批量查询Agent环境变量") + @POST + @Path("/projects/{projectId}/env") + fun fetchAgentEnv( + @Parameter(description = "用户ID", required = true, example = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @Parameter(description = "项目ID", required = true) + @PathParam("projectId") + projectId: String, + @Parameter(description = "查询数据", required = true) + data: BatchFetchAgentData + ): Result>> + + @Operation(summary = "批量修改Agent环境变量") + @POST + @Path("/projects/{projectId}/batch_update_env") + fun batchUpdateEnv( + @Parameter(description = "用户ID", required = true, example = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @Parameter(description = "项目ID", required = true) + @PathParam("projectId") + projectId: String, + @Parameter(description = "修改数据", required = true) + data: BatchUpdateAgentEnvVar + ): Result } diff --git a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/api/thirdpartyagent/UserThirdPartyAgentResource.kt b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/api/thirdpartyagent/UserThirdPartyAgentResource.kt index b92ef3554d3..a417023d9c0 100644 --- a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/api/thirdpartyagent/UserThirdPartyAgentResource.kt +++ b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/api/thirdpartyagent/UserThirdPartyAgentResource.kt @@ -92,6 +92,24 @@ interface UserThirdPartyAgentResource { zoneName: String? ): Result + @Operation(summary = "生成批量安装链接") + @GET + @Path("/projects/{projectId}/os/{os}/generateBatchInstallLink") + fun generateBatchInstallLink( + @Parameter(description = "用户ID", required = true, example = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @Parameter(description = "项目ID", required = true) + @PathParam("projectId") + projectId: String, + @Parameter(description = "操作系统", required = true) + @PathParam("os") + os: OS, + @Parameter(description = "网关地域", required = false) + @QueryParam("zoneName") + zoneName: String? + ): Result + @Operation(summary = "获取网关列表") @GET @Path("/projects/{projectId}/os/{os}/gateway") @@ -203,6 +221,22 @@ interface UserThirdPartyAgentResource { nodeHashId: String ): Result + @Operation(summary = "批量删除第三方构建机") + @DELETE + @Path("/projects/{projectId}/nodes/batch_delete") + fun batchDeleteAgent( + @Parameter(description = "用户ID", required = true, example = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + @BkField(minLength = 1, maxLength = 128) + userId: String, + @Parameter(description = "项目ID", required = true) + @PathParam("projectId") + @BkField(minLength = 1, maxLength = 64) + projectId: String, + @Parameter(description = "Node Hash IDs", required = true) + nodeHashIds: Set + ): Result + @Operation(summary = "保存agent环境变量") @POST @Path("/projects/{projectId}/nodes/{nodeHashId}/envs") diff --git a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/constant/Constants.kt b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/constant/Constants.kt index 2dcc0583cd4..921d0aaaaee 100644 --- a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/constant/Constants.kt +++ b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/constant/Constants.kt @@ -42,6 +42,11 @@ const val T_NODE_AGENT_STATUS = "agentStatus" const val T_NODE_PROJECT_ID = "projectId" const val T_NODE_CREATED_USER = "createdUser" const val T_NODE_OS_TYPE = "osType" +const val T_NODE_OS_NAME = "osName" +const val T_NODE_SERVER_ID = "serverId" +const val T_NODE_OPERATOR = "operator" +const val T_NODE_BAK_OPERATOR = "bakOperator" const val T_ENV_ENV_ID = "envId" const val T_ENVIRONMENT_THIRDPARTY_AGENT_NODE_ID = "nodeId" const val T_ENVIRONMENT_THIRDPARTY_AGENT_MASTER_VERSION = "masterVersion" +const val BATCH_TOKEN_HEADER = "X-DEVOPS-AGENT-INSTALL-TOKEN" // 批量安装agent token的header diff --git a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/constant/EnvironmentMessageCode.kt b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/constant/EnvironmentMessageCode.kt index 80645193811..84fb7969fd1 100644 --- a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/constant/EnvironmentMessageCode.kt +++ b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/constant/EnvironmentMessageCode.kt @@ -94,8 +94,15 @@ object EnvironmentMessageCode { const val ERROR_NODE_NO_USE_PERMISSSION = "2105045" // 环境管理：节点[{0}]没有使用权限 // "2105046" 环境管理: 不在CMDB中的IP [{0}]; 无权限的IP [{1}], 请确认当前用户[{2}]或节点导入人[{3}]是否为这些节点的主备负责人 - 脚本执行｜文件分发 const val ERROR_NODE_IP_ILLEGAL = "2105046" - const val ERROR_CMDB_INTERFACE_TIME_OUT = "2105047" - const val ERROR_CMDB_RESPONSE = "2105048" + const val ERROR_CMDB_INTERFACE_TIME_OUT = "2105047" // 环境管理: CMDB接口请求超时，请重试 + const val ERROR_CMDB_RESPONSE = "2105048" // 环境管理: CMDB接口请求异常，请重试 + const val ERROR_SCRIPT_EXECUTE_HOST_EMPTY = "2105049" // 环境管理: 脚本执行: 主机为空 + const val ERROR_DISTRIBUTE_FILE_EXECUTE_TARGET_HOST_EMPTY = "2105050" // 环境管理: 文件分发: 执行目标主机为空 + const val ERROR_DISTRIBUTE_FILE_FILE_SOURCE_HOST_EMPTY = "2105051" // 环境管理: 文件分发: 文件源主机为空 + const val ERROR_ENV_LIST_NODE_NOT_IN_CC_OR_CMDB = "2105052" // 环境管理: 环境中的[{0}]不在CC/CMDB中 + const val ERROR_NODE_LIST_NODE_NOT_IN_CC_OR_CMDB = "2105053" // 环境管理: 节点中的[{0}]不在CC/CMDB中 + const val ERROR_JOB_INSTANCE_NOT_BELONG_TO_PROJECT = "2105054" // 环境管理: 请求的job实例不属于当前项目或已过期(超过一个月) + const val ERROR_FAIL_TO_CREATE_AGENT_INSTALL_TASK = "2105055" // 环境管理: 创建Agent安装任务失败：{0} const val BK_NORMAL_VERSION = "bkNormalVersion" // 8核16G（普通版） const val BK_INTEL_XEON_SKYLAKE_PROCESSOR = "bkIntelXeonSkylakeProcessor" // 2.5GHz 64核 Intel Xeon Skylake 6133处理器 diff --git a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/CmdbNode.kt b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/CmdbNode.kt index 105308bdcc7..c82e677fc90 100644 --- a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/CmdbNode.kt +++ b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/CmdbNode.kt @@ -46,16 +46,9 @@ data class CmdbNode( @get:Schema(title = "所属业务") val bizId: Long = -1, @get:Schema(title = "节点状态") - var nodeStatus: String?, + var nodeStatus: String? = null, @get:Schema(title = "是否已导入") - var importStatus: Boolean? = false -) { - constructor( - name: String, - operator: String, - bakOperator: String, - ip: String, - displayIp: String, - osName: String - ) : this(name, operator, bakOperator, ip, displayIp, osName, -1L, null, null) -} + var importStatus: Boolean? = false, + @get:Schema(title = "主机id") + val serverId: Long? +) diff --git a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/EnvVar.kt b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/EnvVar.kt index 2ca4b062be1..7bfe5572166 100644 --- a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/EnvVar.kt +++ b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/EnvVar.kt @@ -27,9 +27,11 @@ package com.tencent.devops.environment.pojo +import com.fasterxml.jackson.annotation.JsonIgnoreProperties import io.swagger.v3.oas.annotations.media.Schema @Schema(title = "环境变量") +@JsonIgnoreProperties(ignoreUnknown = true) data class EnvVar( @get:Schema(title = "变量名", required = true) val name: String, diff --git a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/NodeWithPermission.kt b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/NodeWithPermission.kt index 3a935eb64fd..9afc5edb615 100644 --- a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/NodeWithPermission.kt +++ b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/NodeWithPermission.kt @@ -88,5 +88,7 @@ data class NodeWithPermission( @get:Schema(title = "hostID") val bkHostId: Long? = null, @get:Schema(title = "job任务ID") - val taskId: Long? + val taskId: Long?, + @get:Schema(title = "主机serverId") + val serverId: Long? ) diff --git a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/thirdpartyagent/BatchFetchAgentData.kt b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/thirdpartyagent/BatchFetchAgentData.kt new file mode 100644 index 00000000000..bc7153e1eb0 --- /dev/null +++ b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/thirdpartyagent/BatchFetchAgentData.kt @@ -0,0 +1,24 @@ +package com.tencent.devops.environment.pojo.thirdpartyagent + +import com.tencent.devops.environment.pojo.EnvVar +import io.swagger.v3.oas.annotations.media.Schema + +@Schema(title = "批量查询Agent数据") +open class BatchFetchAgentData( + @get:Schema(title = "Node Hash ID列表,和 agentHashId 选其一即可", required = false) + open val nodeHashIds: Set?, + @get:Schema(title = "agent Hash ID列表,和 nodeHashId 选其一即可", required = false) + open val agentHashIds: Set? +) + +@Schema(title = "批量修改Agent环境变量数据") +data class BatchUpdateAgentEnvVar( + @get:Schema(title = "Node Hash ID列表,和 agentHashId 选其一即可", required = false) + override val nodeHashIds: Set?, + @get:Schema(title = "agent Hash ID列表,和 nodeHashId 选其一即可", required = false) + override val agentHashIds: Set?, + @get:Schema(title = "修改方式,支持3种输入(ADD,REMOVE,UPDATE),默认为UPDATE", required = false) + val type: ThirdPartAgentUpdateType?, + @get:Schema(title = "环境变量", required = true) + val envVars: List +) : BatchFetchAgentData(nodeHashIds, agentHashIds) \ No newline at end of file diff --git a/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/thirdpartyagent/ThirdPartAgentUpdateType.kt b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/thirdpartyagent/ThirdPartAgentUpdateType.kt new file mode 100644 index 00000000000..a102fc3637d --- /dev/null +++ b/src/backend/ci/core/environment/api-environment/src/main/kotlin/com/tencent/devops/environment/pojo/thirdpartyagent/ThirdPartAgentUpdateType.kt @@ -0,0 +1,7 @@ +package com.tencent.devops.environment.pojo.thirdpartyagent + +enum class ThirdPartAgentUpdateType { + ADD, + REMOVE, + UPDATE +} \ No newline at end of file diff --git a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/cron/ThirdPartyAgentHeartBeatJob.kt b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/cron/ThirdPartyAgentHeartBeatJob.kt index 712fafab966..1f968ba4a83 100644 --- a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/cron/ThirdPartyAgentHeartBeatJob.kt +++ b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/cron/ThirdPartyAgentHeartBeatJob.kt @@ -110,7 +110,7 @@ class ThirdPartyAgentHeartBeatJob @Autowired constructor( if (nodeRecord == null || nodeRecord.nodeStatus == NodeStatus.DELETED.name) { deleteAgent(context, record.projectId, record.id) } - nodeDao.updateNodeStatus(context, record.nodeId, NodeStatus.ABNORMAL) + nodeDao.updateNodeStatus(context, setOf(record.nodeId), NodeStatus.ABNORMAL) } } } diff --git a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/dao/NodeDao.kt b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/dao/NodeDao.kt index a2e8e7a39c3..58882ef64d5 100644 --- a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/dao/NodeDao.kt +++ b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/dao/NodeDao.kt @@ -329,13 +329,13 @@ class NodeDao { fun updateNodeStatus( dslContext: DSLContext, - id: Long, + ids: Set, status: NodeStatus ) { with(TNode.T_NODE) { dslContext.update(this) .set(NODE_STATUS, status.name) - .where(NODE_ID.eq(id)) + .where(NODE_ID.`in`(ids)) .execute() } } diff --git a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/dao/thirdpartyagent/AgentBatchInstallTokenDao.kt b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/dao/thirdpartyagent/AgentBatchInstallTokenDao.kt new file mode 100644 index 00000000000..4a9086ca767 --- /dev/null +++ b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/dao/thirdpartyagent/AgentBatchInstallTokenDao.kt @@ -0,0 +1,60 @@ +package com.tencent.devops.environment.dao.thirdpartyagent + +import com.tencent.devops.model.environment.tables.TAgentBatchInstallToken +import com.tencent.devops.model.environment.tables.records.TAgentBatchInstallTokenRecord +import org.jooq.DSLContext +import org.springframework.stereotype.Repository +import java.time.LocalDateTime + +@Repository +class AgentBatchInstallTokenDao { + fun createOrUpdateToken( + dslContext: DSLContext, + projectId: String, + userId: String, + token: String, + createTime: LocalDateTime, + expireTime: LocalDateTime + ) { + with(TAgentBatchInstallToken.T_AGENT_BATCH_INSTALL_TOKEN) { + dslContext.insertInto( + this, + PROJECT_ID, + USER_ID, + TOKEN, + CREATED_TIME, + EXPIRED_TIME + ).values( + projectId, + userId, + token, + createTime, + expireTime + ).onDuplicateKeyUpdate() + .set(TOKEN, token) + .set(CREATED_TIME, createTime) + .set(EXPIRED_TIME, expireTime) + .execute() + } + } + + fun deleteToken( + dslContext: DSLContext, + projectId: String, + userId: String + ) { + with(TAgentBatchInstallToken.T_AGENT_BATCH_INSTALL_TOKEN) { + dslContext.deleteFrom(this).where(PROJECT_ID.eq(projectId)).and(USER_ID.eq(userId)).execute() + } + } + + fun getToken( + dslContext: DSLContext, + projectId: String, + userId: String + ): TAgentBatchInstallTokenRecord? { + with(TAgentBatchInstallToken.T_AGENT_BATCH_INSTALL_TOKEN) { + return dslContext.selectFrom(this).where(PROJECT_ID.eq(projectId)).and(USER_ID.eq(userId)).fetchAny() + } + } +} diff --git a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/dao/thirdpartyagent/ThirdPartyAgentDao.kt b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/dao/thirdpartyagent/ThirdPartyAgentDao.kt index f14870c55a4..27d2c197878 100644 --- a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/dao/thirdpartyagent/ThirdPartyAgentDao.kt +++ b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/dao/thirdpartyagent/ThirdPartyAgentDao.kt @@ -239,6 +239,21 @@ class ThirdPartyAgentDao { } } + fun batchUpdateStatus( + dslContext: DSLContext, + projectId: String, + ids: Set, + status: AgentStatus + ): Int { + with(TEnvironmentThirdpartyAgent.T_ENVIRONMENT_THIRDPARTY_AGENT) { + return dslContext.update(this) + .set(STATUS, status.status) + .where(ID.`in`(ids)) + .and(PROJECT_ID.eq(projectId)) + .execute() + } + } + fun updateAgentVersion( dslContext: DSLContext, id: Long, @@ -345,6 +360,19 @@ class ThirdPartyAgentDao { } } + fun getAgentByAgentIds( + dslContext: DSLContext, + ids: Set, + projectId: String + ): List { + with(TEnvironmentThirdpartyAgent.T_ENVIRONMENT_THIRDPARTY_AGENT) { + return dslContext.selectFrom(this) + .where(ID.`in`(ids)) + .and(PROJECT_ID.eq(projectId)) + .fetch() + } + } + fun getAgentByNodeIdAllProj(dslContext: DSLContext, nodeIdList: List): Result> { with(TEnvironmentThirdpartyAgent.T_ENVIRONMENT_THIRDPARTY_AGENT) { return dslContext.select( @@ -400,11 +428,11 @@ class ThirdPartyAgentDao { } } - fun saveAgentEnvs(dslContext: DSLContext, agentId: Long, envStr: String) { + fun saveAgentEnvs(dslContext: DSLContext, agentIds: Set, envStr: String) { with(TEnvironmentThirdpartyAgent.T_ENVIRONMENT_THIRDPARTY_AGENT) { dslContext.update(this) .set(AGENT_ENVS, envStr) - .where(ID.eq(agentId)) + .where(ID.`in`(agentIds)) .execute() } } diff --git a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/model/CreateNodeModel.kt b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/model/CreateNodeModel.kt index 32ea37df213..96ef40976f7 100644 --- a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/model/CreateNodeModel.kt +++ b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/model/CreateNodeModel.kt @@ -33,7 +33,7 @@ data class CreateNodeModel( var nodeStringId: String? = "", var projectId: String, var nodeIp: String = "", - var nodeName: String = "", + var nodeName: String? = "", var nodeStatus: String = "", var nodeType: String = "", var nodeClusterId: String? = null, diff --git a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/resources/thirdPartyAgent/ExternalThirdPartyAgentResourceImpl.kt b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/resources/thirdPartyAgent/ExternalThirdPartyAgentResourceImpl.kt index a335d72ded7..9b099967027 100644 --- a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/resources/thirdPartyAgent/ExternalThirdPartyAgentResourceImpl.kt +++ b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/resources/thirdPartyAgent/ExternalThirdPartyAgentResourceImpl.kt @@ -27,9 +27,11 @@ package com.tencent.devops.environment.resources.thirdpartyagent +import com.tencent.devops.common.api.pojo.OS +import com.tencent.devops.common.api.pojo.agent.AgentArchType import com.tencent.devops.common.web.RestResource import com.tencent.devops.environment.api.thirdpartyagent.ExternalThirdPartyAgentResource -import com.tencent.devops.common.api.pojo.agent.AgentArchType +import com.tencent.devops.environment.service.thirdpartyagent.BatchInstallAgentService import com.tencent.devops.environment.service.thirdpartyagent.DownloadAgentInstallService import com.tencent.devops.environment.service.thirdpartyagent.ImportService import org.springframework.beans.factory.annotation.Autowired @@ -38,10 +40,11 @@ import javax.ws.rs.core.Response @RestResource class ExternalThirdPartyAgentResourceImpl @Autowired constructor( private val downloadAgentInstallService: DownloadAgentInstallService, - private val importService: ImportService + private val importService: ImportService, + private val batchInstallAgentService: BatchInstallAgentService ) : ExternalThirdPartyAgentResource { override fun downloadAgentInstallScript(agentId: String) = - downloadAgentInstallService.downloadInstallScript(agentId) + downloadAgentInstallService.downloadInstallScript(agentId, false) override fun downloadAgent(agentId: String, eTag: String?, arch: String?) = downloadAgentInstallService.downloadAgent( @@ -55,7 +58,8 @@ class ExternalThirdPartyAgentResourceImpl @Autowired constructor( override fun downloadJRE(agentId: String, eTag: String?, arch: String?) = downloadAgentInstallService.downloadJre( - agentId, eTag, arch = when (arch) { + agentId, eTag, + arch = when (arch) { "arm64" -> AgentArchType.ARM64 "mips64" -> AgentArchType.MIPS64 else -> null @@ -66,4 +70,12 @@ class ExternalThirdPartyAgentResourceImpl @Autowired constructor( val newAgentId = importService.generateAgentByOtherAgentId(agentHashId) return downloadAgentInstallService.downloadInstallAgentBatchFile(newAgentId) } + + override fun batchDownloadAgentInstallScript(token: String, os: OS, zoneName: String?): Response { + return batchInstallAgentService.genAgentInstallScript( + token = token, + os = os, + zoneName = zoneName + ) + } } diff --git a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/resources/thirdPartyAgent/ServiceThirdPartyAgentResourceImpl.kt b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/resources/thirdPartyAgent/ServiceThirdPartyAgentResourceImpl.kt index 263c619a064..44e258d8fbd 100644 --- a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/resources/thirdPartyAgent/ServiceThirdPartyAgentResourceImpl.kt +++ b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/resources/thirdPartyAgent/ServiceThirdPartyAgentResourceImpl.kt @@ -40,11 +40,14 @@ import com.tencent.devops.environment.api.thirdpartyagent.ServiceThirdPartyAgent import com.tencent.devops.environment.constant.EnvironmentMessageCode import com.tencent.devops.environment.permission.EnvironmentPermissionService import com.tencent.devops.environment.pojo.AgentPipelineRefRequest +import com.tencent.devops.environment.pojo.EnvVar import com.tencent.devops.environment.pojo.enums.NodeType import com.tencent.devops.environment.pojo.slave.SlaveGateway import com.tencent.devops.environment.pojo.thirdpartyagent.AgentBuildDetail import com.tencent.devops.environment.pojo.thirdpartyagent.AgentPipelineRef import com.tencent.devops.environment.pojo.thirdpartyagent.AskHeartbeatResponse +import com.tencent.devops.environment.pojo.thirdpartyagent.BatchFetchAgentData +import com.tencent.devops.environment.pojo.thirdpartyagent.BatchUpdateAgentEnvVar import com.tencent.devops.environment.pojo.thirdpartyagent.EnvNodeAgent import com.tencent.devops.environment.pojo.thirdpartyagent.ThirdPartyAgent import com.tencent.devops.environment.pojo.thirdpartyagent.ThirdPartyAgentDetail @@ -57,6 +60,7 @@ import com.tencent.devops.environment.pojo.thirdpartyagent.pipeline.PipelineSeqI import com.tencent.devops.environment.service.NodeService import com.tencent.devops.environment.service.slave.SlaveGatewayService import com.tencent.devops.environment.service.thirdpartyagent.AgentPipelineService +import com.tencent.devops.environment.service.thirdpartyagent.ThirdPartAgentService import com.tencent.devops.environment.service.thirdpartyagent.ThirdPartyAgentMgrService import com.tencent.devops.environment.service.thirdpartyagent.ThirdPartyAgentPipelineService import com.tencent.devops.environment.service.thirdpartyagent.UpgradeService @@ -70,7 +74,8 @@ class ServiceThirdPartyAgentResourceImpl @Autowired constructor( private val agentPipelineService: AgentPipelineService, private val slaveGatewayService: SlaveGatewayService, private val permissionService: EnvironmentPermissionService, - private val nodeService: NodeService + private val nodeService: NodeService, + private val agentService: ThirdPartAgentService ) : ServiceThirdPartyAgentResource { override fun getAgentById(projectId: String, agentId: String): AgentResult { return thirdPartyAgentService.getAgent(projectId, agentId) @@ -277,4 +282,36 @@ class ServiceThirdPartyAgentResourceImpl @Autowired constructor( ): Result>> { return Result(thirdPartyAgentService.getAgentByEnvName(projectId, envName)) } + + override fun fetchAgentEnv( + userId: String, + projectId: String, + data: BatchFetchAgentData + ): Result>> { + return Result( + agentService.fetchAgentEnv( + userId = userId, + projectId = projectId, + nodeHashIds = data.nodeHashIds, + agentHashIds = data.agentHashIds + ) + ) + } + + override fun batchUpdateEnv( + userId: String, + projectId: String, + data: BatchUpdateAgentEnvVar + ): Result { + return Result( + agentService.batchUpdateAgentEnv( + userId = userId, + projectId = projectId, + nodeHashIds = data.nodeHashIds, + agentHashIds = data.agentHashIds, + type = data.type, + data = data.envVars + ) + ) + } } diff --git a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/resources/thirdPartyAgent/UserThirdPartyAgentResourceImpl.kt b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/resources/thirdPartyAgent/UserThirdPartyAgentResourceImpl.kt index 98056a13380..420d300cc90 100644 --- a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/resources/thirdPartyAgent/UserThirdPartyAgentResourceImpl.kt +++ b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/resources/thirdPartyAgent/UserThirdPartyAgentResourceImpl.kt @@ -47,6 +47,7 @@ import com.tencent.devops.environment.pojo.thirdpartyagent.ThirdPartyAgentStatus import com.tencent.devops.environment.service.slave.SlaveGatewayService import com.tencent.devops.environment.service.thirdpartyagent.AgentMetricService import com.tencent.devops.environment.service.thirdpartyagent.ImportService +import com.tencent.devops.environment.service.thirdpartyagent.BatchInstallAgentService import com.tencent.devops.environment.service.thirdpartyagent.ThirdPartyAgentMgrService import org.springframework.beans.factory.annotation.Autowired @@ -56,7 +57,8 @@ class UserThirdPartyAgentResourceImpl @Autowired constructor( private val thirdPartyAgentService: ThirdPartyAgentMgrService, private val slaveGatewayService: SlaveGatewayService, private val importService: ImportService, - private val agentMetricService: AgentMetricService + private val agentMetricService: AgentMetricService, + private val batchInstallAgentService: BatchInstallAgentService ) : UserThirdPartyAgentResource { override fun isProjectEnable(userId: String, projectId: String): Result { return Result(true) @@ -73,6 +75,24 @@ class UserThirdPartyAgentResourceImpl @Autowired constructor( return Result(thirdPartyAgentService.generateAgent(userId, projectId, os, zoneName)) } + override fun generateBatchInstallLink( + userId: String, + projectId: String, + os: OS, + zoneName: String? + ): Result { + checkUserId(userId) + checkProjectId(projectId) + return Result( + batchInstallAgentService.genInstallLink( + projectId = projectId, + userId = userId, + os = os, + zoneName = zoneName + ) + ) + } + override fun getGateway( userId: String, projectId: String, @@ -122,7 +142,12 @@ class UserThirdPartyAgentResourceImpl @Autowired constructor( @AuditEntry(actionId = ActionId.ENV_NODE_DELETE) override fun deleteAgent(userId: String, projectId: String, nodeHashId: String): Result { - thirdPartyAgentService.deleteAgent(userId, projectId, nodeHashId) + thirdPartyAgentService.deleteAgent(userId, projectId, setOf(nodeHashId)) + return Result(true) + } + + override fun batchDeleteAgent(userId: String, projectId: String, nodeHashIds: Set): Result { + thirdPartyAgentService.deleteAgent(userId, projectId, nodeHashIds) return Result(true) } diff --git a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/AgentUrlService.kt b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/AgentUrlService.kt index fcea1ef121e..a178d058994 100644 --- a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/AgentUrlService.kt +++ b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/AgentUrlService.kt @@ -27,11 +27,13 @@ package com.tencent.devops.environment.service +import com.tencent.devops.common.api.pojo.OS import com.tencent.devops.model.environment.tables.records.TEnvironmentThirdpartyAgentRecord interface AgentUrlService { fun genAgentInstallUrl(agentRecord: TEnvironmentThirdpartyAgentRecord): String + /** *生成Agent URL */ @@ -42,6 +44,16 @@ interface AgentUrlService { */ fun genAgentInstallScript(agentRecord: TEnvironmentThirdpartyAgentRecord): String + /** + * 生成批量下载构建机脚本链接 + */ + fun genAgentBatchInstallScript( + os: OS, + zoneName: String?, + gateway: String?, + token: String + ): String + /** * 生成网关域名 */ diff --git a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/BluekingAgentUrlServiceImpl.kt b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/BluekingAgentUrlServiceImpl.kt index c4cc7c2b26c..9afe7c57591 100644 --- a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/BluekingAgentUrlServiceImpl.kt +++ b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/BluekingAgentUrlServiceImpl.kt @@ -31,6 +31,7 @@ import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_PROJECT_ID import com.tencent.devops.common.api.pojo.OS import com.tencent.devops.common.api.util.HashUtil import com.tencent.devops.common.service.config.CommonConfig +import com.tencent.devops.environment.constant.BATCH_TOKEN_HEADER import com.tencent.devops.model.environment.tables.records.TEnvironmentThirdpartyAgentRecord /** @@ -66,6 +67,28 @@ class BluekingAgentUrlServiceImpl constructor( } } + override fun genAgentBatchInstallScript( + os: OS, + zoneName: String?, + gateway: String?, + token: String + ): String { + val gw = fixGateway(gateway) + if (os == OS.WINDOWS) { + return "\$headers = @{ \"$BATCH_TOKEN_HEADER\" = \"$token\" }; " + + "\$response = Invoke-WebRequest " + + "-Uri \"$gw/ms/environment/api/external/thirdPartyAgent/${os.name}/batchInstall\" " + + "-Headers \$headers; " + + "\$ps = [System.Text.Encoding]::UTF8.GetString(\$response.Content);Invoke-Expression -Command \$ps" + } + var url = "curl -H \"$BATCH_TOKEN_HEADER: $token\" " + + "$gw/ms/environment/api/external/thirdPartyAgent/${os.name}/batchInstall" + if (!zoneName.isNullOrBlank()) { + url += "?zoneName=$zoneName" + } + return "$url | bash" + } + override fun genGateway(agentRecord: TEnvironmentThirdpartyAgentRecord): String { return fixGateway(agentRecord.gateway) } diff --git a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/NodeService.kt b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/NodeService.kt index c9befe0c9d9..535fce61b2f 100644 --- a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/NodeService.kt +++ b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/NodeService.kt @@ -296,7 +296,8 @@ class NodeService @Autowired constructor( } else { it.osType }, - bkHostId = it.hostId + bkHostId = it.hostId, + serverId = it.serverId ) } } @@ -387,7 +388,8 @@ class NodeService @Autowired constructor( agentHashId = HashUtil.encodeLongId(thirdPartyAgent?.id ?: 0L), cloudAreaId = it.cloudAreaId, taskId = null, - osType = it.osType + osType = it.osType, + serverId = it.serverId ) } } @@ -443,7 +445,8 @@ class NodeService @Autowired constructor( lastModifyUser = it.lastModifyUser ?: "", cloudAreaId = it.cloudAreaId, taskId = null, - osType = it.osType + osType = it.osType, + serverId = it.serverId ) } } @@ -634,7 +637,8 @@ class NodeService @Autowired constructor( }, cloudAreaId = it.cloudAreaId, taskId = null, - osType = it.osType + osType = it.osType, + serverId = it.serverId ) } } diff --git a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/slave/SlaveGatewayService.kt b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/slave/SlaveGatewayService.kt index 6479ee65c24..3a1d52e9319 100644 --- a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/slave/SlaveGatewayService.kt +++ b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/slave/SlaveGatewayService.kt @@ -87,7 +87,10 @@ class SlaveGatewayService @Autowired constructor( // @return gateway,filegateway private fun getConfigGateway(zoneName: String?): Pair { if (zoneName.isNullOrBlank()) { - return Pair(agentUrlService.fixGateway(commonConfig.devopsBuildGateway!!), null) + return Pair( + agentUrlService.fixGateway(commonConfig.devopsBuildGateway!!), + agentUrlService.fixGateway(commonConfig.fileDevnetGateway!!) + ) } val gateways = getGateway() gateways.forEach { @@ -98,7 +101,10 @@ class SlaveGatewayService @Autowired constructor( ) } } - return Pair(agentUrlService.fixGateway(commonConfig.devopsBuildGateway!!), null) + return Pair( + agentUrlService.fixGateway(commonConfig.devopsBuildGateway!!), + agentUrlService.fixGateway(commonConfig.fileDevnetGateway!!) + ) } fun getGateway(): List { diff --git a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/BatchInstallAgentService.kt b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/BatchInstallAgentService.kt new file mode 100644 index 00000000000..0c3fbdd9950 --- /dev/null +++ b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/BatchInstallAgentService.kt @@ -0,0 +1,151 @@ +package com.tencent.devops.environment.service.thirdpartyagent + +import com.tencent.devops.common.api.exception.OperationException +import com.tencent.devops.common.api.pojo.OS +import com.tencent.devops.common.api.util.AESUtil +import com.tencent.devops.common.api.util.ApiUtil +import com.tencent.devops.common.api.util.HashUtil +import com.tencent.devops.common.api.util.SecurityUtil +import com.tencent.devops.common.redis.concurrent.SimpleRateLimiter +import com.tencent.devops.environment.dao.thirdpartyagent.AgentBatchInstallTokenDao +import com.tencent.devops.environment.dao.thirdpartyagent.ThirdPartyAgentDao +import com.tencent.devops.environment.service.AgentUrlService +import com.tencent.devops.environment.service.slave.SlaveGatewayService +import org.jooq.DSLContext +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service +import java.time.LocalDateTime +import java.time.ZoneOffset +import javax.ws.rs.core.Response + +/** + * 批量安装Agent相关 + */ +@Service +class BatchInstallAgentService @Autowired constructor( + private val dslContext: DSLContext, + private val agentBatchInstallTokenDao: AgentBatchInstallTokenDao, + private val thirdPartyAgentDao: ThirdPartyAgentDao, + private val agentUrlService: AgentUrlService, + private val slaveGatewayService: SlaveGatewayService, + private val downloadAgentInstallService: DownloadAgentInstallService, + private val simpleRateLimiter: SimpleRateLimiter +) { + fun genInstallLink( + projectId: String, + userId: String, + os: OS, + zoneName: String? + ): String { + val now = LocalDateTime.now() + val gateway = slaveGatewayService.getGateway(zoneName) + // 先确定下是否已经生成过了，以及有没有过期 + val record = agentBatchInstallTokenDao.getToken( + dslContext = dslContext, + projectId = projectId, + userId = userId + ) + if (record != null && record.expiredTime > now) { + return agentUrlService.genAgentBatchInstallScript( + os = os, + zoneName = zoneName, + gateway = gateway, + token = record.token + ) + } + + // 没有或者过期则重新生成，过期时间默认为3天后 + val tokenData = "$projectId;$userId;${now.toInstant(ZoneOffset.of("+8")).toEpochMilli()}" + logger.debug("genInstallLink token data $tokenData") + val token = AESUtil.encrypt(ASE_SECRET, tokenData) + val expireTime = now.plusDays(3) + agentBatchInstallTokenDao.createOrUpdateToken( + dslContext = dslContext, + projectId = projectId, + userId = userId, + token = token, + createTime = now, + expireTime = expireTime + ) + + return agentUrlService.genAgentBatchInstallScript( + os = os, + zoneName = zoneName, + gateway = gateway, + token = token + ) + } + + fun genAgentInstallScript( + token: String, + os: OS, + zoneName: String? + ): Response { + // 先校验是否可以创建 + val (projectId, userId, errorMsg) = verifyToken(token) + if (errorMsg != null) { + throw RuntimeException(errorMsg) + } + + // 增加下载限制 + val lockKey = "lock:tpa:batch:rate:$token" + val acquire = simpleRateLimiter.acquire(ImportService.BU_SIZE, lockKey = lockKey) + if (!acquire) { + throw OperationException("Frequency $lockKey limit: ${ImportService.BU_SIZE}") + } + + // 直接创建新agent + val agentId = genNewAgent( + projectId = projectId, + userId = userId, + os = os, + zoneName = zoneName + ) + val agentHashId = HashUtil.encodeLongId(agentId) + + // 生成安装脚本 + return downloadAgentInstallService.downloadInstallScript(agentHashId, true) + } + + private fun verifyToken(token: String): Triple { + val decodeSub = AESUtil.decrypt(ASE_SECRET, token).split(";") + if (decodeSub.size < 3) { + return Triple("", "", "token verify error") + } + + val record = agentBatchInstallTokenDao.getToken(dslContext, decodeSub[0], decodeSub[1]) + ?: return Triple("", "", "token's project and user not find") + + if (record.token != token || record.expiredTime <= LocalDateTime.now()) { + return Triple("", "", "token is expired") + } + + return Triple(decodeSub[0], decodeSub[1], null) + } + + private fun genNewAgent( + projectId: String, + userId: String, + os: OS, + zoneName: String? + ): Long { + val gateway = slaveGatewayService.getGateway(zoneName) + val fileGateway = slaveGatewayService.getFileGateway(zoneName) + val secretKey = ApiUtil.randomSecretKey() + return thirdPartyAgentDao.add( + dslContext = dslContext, + userId = userId, + projectId = projectId, + os = os, + secretKey = SecurityUtil.encrypt(secretKey), + gateway = gateway, + fileGateway = fileGateway + ) + } + + companion object { + private const val ASE_SECRET = "6fQyK-&Ht49zlBwhB8TW*xAJ/JZz0ZreVcDVCSj+5bY=" + private val logger = LoggerFactory.getLogger(BatchInstallAgentService::class.java) + } +} diff --git a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/DownloadAgentInstallService.kt b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/DownloadAgentInstallService.kt index a5c8a7a054e..1aad8272f74 100644 --- a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/DownloadAgentInstallService.kt +++ b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/DownloadAgentInstallService.kt @@ -76,10 +76,14 @@ class DownloadAgentInstallService @Autowired constructor( @Value("\${environment.certFilePath:#{null}}") private val certFilePath: String? = null - fun downloadInstallScript(agentId: String): Response { + fun downloadInstallScript(agentId: String, isWinDownload: Boolean): Response { logger.info("Trying to download the agent($agentId) install script") val agentRecord = getAgentRecord(agentId) + if (agentRecord.status == AgentStatus.IMPORT_OK.status) { + throw RuntimeException("Agent already installed. Please obtain the install url again") + } + /** * agent_url * jre_url @@ -89,11 +93,15 @@ class DownloadAgentInstallService @Autowired constructor( * gateWay */ val fileName = if (agentRecord.os == OS.WINDOWS.name) { - "install.bat" + if (isWinDownload) { + "download_install.ps1" + } else { + "install.bat" + } } else { "install.sh" } - val scriptFile = File(agentPackage, "script/${agentRecord.os.toLowerCase()}/$fileName") + val scriptFile = File(agentPackage, "script/${agentRecord.os.lowercase()}/$fileName") if (!scriptFile.exists()) { logger.warn("The install script file(${scriptFile.absolutePath}) is not exist") diff --git a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/ImportService.kt b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/ImportService.kt index 598b1fd1cc4..b57463168b1 100644 --- a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/ImportService.kt +++ b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/ImportService.kt @@ -75,7 +75,7 @@ class ImportService @Autowired constructor( ) { companion object { - private const val BU_SIZE = 100 + const val BU_SIZE = 100 private val LOG = LoggerFactory.getLogger(ImportService::class.java) private val badStatus = setOf(AgentStatus.IMPORT_EXCEPTION.status, AgentStatus.UN_IMPORT.status) } diff --git a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/ThirdPartAgentService.kt b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/ThirdPartAgentService.kt index f1a5e13701c..a0a0638085c 100644 --- a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/ThirdPartAgentService.kt +++ b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/ThirdPartAgentService.kt @@ -1,9 +1,16 @@ package com.tencent.devops.environment.service.thirdpartyagent +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.kotlin.readValue import com.tencent.devops.common.api.enums.AgentAction +import com.tencent.devops.common.api.util.HashUtil import com.tencent.devops.common.redis.RedisOperation import com.tencent.devops.environment.dao.thirdpartyagent.ThirdPartyAgentActionDao +import com.tencent.devops.environment.dao.thirdpartyagent.ThirdPartyAgentDao +import com.tencent.devops.environment.pojo.EnvVar +import com.tencent.devops.environment.pojo.thirdpartyagent.ThirdPartAgentUpdateType import com.tencent.devops.environment.utils.ThirdAgentActionAddLock +import com.tencent.devops.environment.utils.ThirdAgentUpdateEnvLock import org.jooq.DSLContext import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service @@ -11,11 +18,14 @@ import org.springframework.stereotype.Service /** * 对第三方构建机一些自身数据操作 */ +@Suppress("NestedBlockDepth") @Service class ThirdPartAgentService @Autowired constructor( + private val objectMapper: ObjectMapper, private val redisOperation: RedisOperation, private val dslContext: DSLContext, - private val agentActionDao: ThirdPartyAgentActionDao + private val agentActionDao: ThirdPartyAgentActionDao, + private val agentDao: ThirdPartyAgentDao ) { fun addAgentAction( projectId: String, @@ -41,4 +51,109 @@ class ThirdPartAgentService @Autowired constructor( lock.unlock() } } + + fun fetchAgentEnv( + userId: String, + projectId: String, + nodeHashIds: Set?, + agentHashIds: Set? + ): Map> { + if (agentHashIds.isNullOrEmpty() && nodeHashIds.isNullOrEmpty()) { + return emptyMap() + } + + val res = mutableMapOf>() + if (!agentHashIds.isNullOrEmpty()) { + val idMap = agentHashIds.associateBy { HashUtil.decodeIdToLong(it) } + val agents = agentDao.getAgentByAgentIds(dslContext, idMap.keys, projectId) + agents.forEach { agent -> + res[idMap[agent.id] ?: return@forEach] = if (agent.agentEnvs.isNullOrBlank()) { + emptyList() + } else { + objectMapper.readValue>(agent.agentEnvs) + } + } + } else { + val idMap = nodeHashIds!!.associateBy { HashUtil.decodeIdToLong(it) } + val agents = agentDao.getAgentsByNodeIds(dslContext, idMap.keys, projectId) + agents.forEach { agent -> + res[idMap[agent.nodeId] ?: return@forEach] = if (agent.agentEnvs.isNullOrBlank()) { + emptyList() + } else { + objectMapper.readValue>(agent.agentEnvs) + } + } + } + return res + } + + fun batchUpdateAgentEnv( + userId: String, + projectId: String, + nodeHashIds: Set?, + agentHashIds: Set?, + type: ThirdPartAgentUpdateType?, + data: List + ): Boolean { + if (agentHashIds.isNullOrEmpty() && nodeHashIds.isNullOrEmpty()) { + return false + } + + val agents = if (!agentHashIds.isNullOrEmpty()) { + agentDao.getAgentByAgentIds(dslContext, agentHashIds.map { HashUtil.decodeIdToLong(it) }.toSet(), projectId) + } else { + agentDao.getAgentsByNodeIds( + dslContext, + nodeHashIds!!.map { HashUtil.decodeIdToLong(it) }.toSet(), + projectId + ) + }.ifEmpty { return false } + + when (type ?: ThirdPartAgentUpdateType.UPDATE) { + ThirdPartAgentUpdateType.ADD, ThirdPartAgentUpdateType.REMOVE -> { + agents.forEach { agent -> + val agentId = agent.id + val lock = ThirdAgentUpdateEnvLock(redisOperation, projectId, agentId) + try { + lock.lock() + val oldEnvsMap = if (agent.agentEnvs.isNullOrBlank()) { + mutableMapOf() + } else { + objectMapper.readValue>(agent.agentEnvs).associateBy { it.name }.toMutableMap() + } + val newEnvsMap = data.associateBy { it.name }.toMutableMap() + val envs = if (type == ThirdPartAgentUpdateType.ADD) { + oldEnvsMap.putAll(newEnvsMap) + oldEnvsMap.values + } else { + newEnvsMap.keys.forEach { key -> + oldEnvsMap.remove(key) + } + oldEnvsMap.values + } + agentDao.saveAgentEnvs( + dslContext = dslContext, + agentIds = agents.map { it.id }.toSet(), + envStr = objectMapper.writeValueAsString(envs) + ) + return true + } finally { + lock.unlock() + } + } + } + + ThirdPartAgentUpdateType.UPDATE -> { + agentDao.saveAgentEnvs( + dslContext = dslContext, + agentIds = agents.map { it.id }.toSet(), + envStr = objectMapper.writeValueAsString(data) + ) + return true + } + + else -> return false + } + return true + } } \ No newline at end of file diff --git a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/ThirdPartyAgentMgrService.kt b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/ThirdPartyAgentMgrService.kt index 4961ce23457..8be997225f0 100644 --- a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/ThirdPartyAgentMgrService.kt +++ b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/service/thirdpartyagent/ThirdPartyAgentMgrService.kt @@ -284,7 +284,7 @@ class ThirdPartyAgentMgrService @Autowired(required = false) constructor( ) thirdPartyAgentDao.saveAgentEnvs( dslContext = dslContext, - agentId = agentRecord.id, + agentIds = setOf(agentRecord.id), envStr = objectMapper.writeValueAsString(envs) ) } @@ -982,34 +982,35 @@ class ThirdPartyAgentMgrService @Autowired(required = false) constructor( fun deleteAgent( userId: String, projectId: String, - nodeId: String + nodeHashIds: Set ) { - logger.info("Delete the node($nodeId) of project($projectId) by user($userId)") - val id = HashUtil.decodeIdToLong(nodeId) + logger.info("Delete the node($nodeHashIds) of project($projectId) by user($userId)") + val nodeIds = nodeHashIds.map { HashUtil.decodeIdToLong(it) }.toSet() dslContext.transaction { configuration -> val context = DSL.using(configuration) - val record = thirdPartyAgentDao.getAgentByNodeId(dslContext = context, nodeId = id, projectId = projectId) - if (record == null) { - logger.warn("The node($nodeId) is not exist") + val records = thirdPartyAgentDao.getAgentsByNodeIds(context, nodeIds, projectId) + if (records.isEmpty()) { + logger.warn("The node($nodeIds) is not exist") throw NotFoundException("The node is not exist") } ActionAuditContext.current() - .setInstanceId(record.nodeId.toString()) - .setInstanceName(record.nodeId.toString()) - val count = thirdPartyAgentDao.updateStatus( + .setInstanceId(nodeIds.joinToString(",")) + .setInstanceName(nodeIds.joinToString(",")) + val count = thirdPartyAgentDao.batchUpdateStatus( dslContext = context, - id = record.id, - nodeId = null, + ids = records.map { it.id }.toSet(), projectId = projectId, status = AgentStatus.DELETE ) - if (count != 1) { + if (count < 1) { logger.warn("Can't delete the agent($count)") } - nodeDao.updateNodeStatus(dslContext = context, id = id, status = NodeStatus.DELETED) - if (record.nodeId != null) { - environmentPermissionService.deleteNode(projectId = projectId, nodeId = record.nodeId) + nodeDao.updateNodeStatus(dslContext = context, ids = nodeIds, status = NodeStatus.DELETED) + records.forEach { record -> + if (record.nodeId != null) { + environmentPermissionService.deleteNode(projectId = projectId, nodeId = record.nodeId) + } } } } @@ -1390,7 +1391,7 @@ class ThirdPartyAgentMgrService @Autowired(required = false) constructor( thirdPartyAgentDao.updateStatus(context, agentRecord.id, null, projectId, AgentStatus.IMPORT_OK) thirdPartAgentService.addAgentAction(projectId, agentRecord.id, AgentAction.ONLINE) if (agentRecord.nodeId != null) { - nodeDao.updateNodeStatus(context, agentRecord.nodeId, NodeStatus.NORMAL) + nodeDao.updateNodeStatus(context, setOf(agentRecord.nodeId), NodeStatus.NORMAL) } AgentStatus.IMPORT_OK } @@ -1404,7 +1405,7 @@ class ThirdPartyAgentMgrService @Autowired(required = false) constructor( return@transactionResult AgentStatus.DELETE } if (nodeRecord.nodeStatus == NodeStatus.ABNORMAL.name) { - val count = nodeDao.updateNodeStatus(context, agentRecord.nodeId, NodeStatus.NORMAL) + val count = nodeDao.updateNodeStatus(context, setOf(agentRecord.nodeId), NodeStatus.NORMAL) agentDisconnectNotifyService?.online( projectId = agentRecord.projectId ?: "", ip = agentRecord.ip ?: "", diff --git a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/utils/ThirdAgentActionAddLock.kt b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/utils/ThirdAgentActionAddLock.kt index 36a12393d2b..84cddcaed3e 100644 --- a/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/utils/ThirdAgentActionAddLock.kt +++ b/src/backend/ci/core/environment/biz-environment/src/main/kotlin/com/tencent/devops/environment/utils/ThirdAgentActionAddLock.kt @@ -11,4 +11,15 @@ class ThirdAgentActionAddLock( redisOperation, "environment:thirdparty:agent.$projectId.$agentId.action.lock", expiredTimeInSeconds = 60 +) + +class ThirdAgentUpdateEnvLock( + redisOperation: RedisOperation, + projectId: String, + agentId: Long +) : RedisLock( + redisOperation, + "environment:thirdparty:agent.$projectId.$agentId.updateenv.lock", + expiredTimeInSeconds = 60, + sleepTime = 10 ) \ No newline at end of file diff --git a/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/log/api/ServiceLogResource.kt b/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/log/api/ServiceLogResource.kt index 38b063dbe70..fceee1806f5 100644 --- a/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/log/api/ServiceLogResource.kt +++ b/src/backend/ci/core/log/api-log/src/main/kotlin/com/tencent/devops/log/api/ServiceLogResource.kt @@ -89,6 +89,9 @@ interface ServiceLogResource { @Parameter(description = "执行次数", required = false) @QueryParam("executeCount") executeCount: Int?, + @Parameter(description = "指定subTag", required = false) + @QueryParam("subTag") + subTag: String? = null, @Parameter(description = "对应jobId", required = false) @QueryParam("jobId") jobId: String?, diff --git a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/resources/ServiceLogResourceImpl.kt b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/resources/ServiceLogResourceImpl.kt index 281d6b9b92d..74eb5b423e3 100644 --- a/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/resources/ServiceLogResourceImpl.kt +++ b/src/backend/ci/core/log/biz-log/src/main/kotlin/com/tencent/devops/log/resources/ServiceLogResourceImpl.kt @@ -61,6 +61,7 @@ class ServiceLogResourceImpl @Autowired constructor( tag: String?, containerHashId: String?, executeCount: Int?, + subTag: String?, jobId: String?, stepId: String?, archiveFlag: Boolean? @@ -75,6 +76,7 @@ class ServiceLogResourceImpl @Autowired constructor( tag = tag, containerHashId = containerHashId, executeCount = executeCount, + subTag = subTag, jobId = jobId, stepId = stepId, archiveFlag = archiveFlag diff --git a/src/backend/ci/core/metrics/api-metrics/build.gradle.kts b/src/backend/ci/core/metrics/api-metrics/build.gradle.kts index c11b0a0f011..2f7765c8e71 100644 --- a/src/backend/ci/core/metrics/api-metrics/build.gradle.kts +++ b/src/backend/ci/core/metrics/api-metrics/build.gradle.kts @@ -29,6 +29,7 @@ dependencies { api(project(":core:common:common-api")) api(project(":core:common:common-web")) api(project(":core:common:common-event")) + api(project(":core:common:common-pipeline")) } plugins { diff --git a/src/backend/ci/core/worker/worker-agent/src/main/kotlin/com/tencent/devops/agent/service/SampleRepoServiceImpl.kt b/src/backend/ci/core/metrics/api-metrics/src/main/kotlin/com/tencent/devops/metrics/pojo/po/MetricsLocalPO.kt similarity index 75% rename from src/backend/ci/core/worker/worker-agent/src/main/kotlin/com/tencent/devops/agent/service/SampleRepoServiceImpl.kt rename to src/backend/ci/core/metrics/api-metrics/src/main/kotlin/com/tencent/devops/metrics/pojo/po/MetricsLocalPO.kt index 1ef3a6fdffc..5262000a8da 100644 --- a/src/backend/ci/core/worker/worker-agent/src/main/kotlin/com/tencent/devops/agent/service/SampleRepoServiceImpl.kt +++ b/src/backend/ci/core/metrics/api-metrics/src/main/kotlin/com/tencent/devops/metrics/pojo/po/MetricsLocalPO.kt @@ -25,22 +25,13 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.agent.service +package com.tencent.devops.metrics.pojo.po -import com.tencent.bkrepo.repository.pojo.token.TokenType -import com.tencent.devops.worker.common.service.RepoService +import io.micrometer.core.instrument.Meter -class SampleRepoServiceImpl : RepoService { - - override fun getRepoToken( - userId: String, - projectId: String, - repoName: String, - path: String, - type: TokenType, - expireSeconds: Long? - ): String? { - // 开源版暂不支持用token去上传或下载 - return null - } +data class MetricsLocalPO( + var data: MetricsUserPO, + val meters: MutableList +) { + constructor(data: MetricsUserPO) : this(data, mutableListOf()) } diff --git a/src/backend/ci/core/metrics/api-metrics/src/main/kotlin/com/tencent/devops/metrics/pojo/po/MetricsUserPO.kt b/src/backend/ci/core/metrics/api-metrics/src/main/kotlin/com/tencent/devops/metrics/pojo/po/MetricsUserPO.kt new file mode 100644 index 00000000000..2d83cd143b3 --- /dev/null +++ b/src/backend/ci/core/metrics/api-metrics/src/main/kotlin/com/tencent/devops/metrics/pojo/po/MetricsUserPO.kt @@ -0,0 +1,97 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.metrics.pojo.po + +import com.tencent.devops.common.event.pojo.pipeline.PipelineBuildStatusBroadCastEvent +import com.tencent.devops.common.pipeline.event.CallBackEvent +import com.tencent.devops.common.pipeline.utils.EventUtils.toEventType +import java.time.Instant +import java.time.LocalDateTime +import java.time.ZoneOffset + +data class MetricsUserPO( + var startTime: LocalDateTime, + val projectId: String, + val pipelineId: String, + val buildId: String, + val jobId: String?, + val stepId: String?, + val status: String, + val atomCode: String?, + val eventType: CallBackEvent, + var endTime: LocalDateTime? +) { + constructor(event: PipelineBuildStatusBroadCastEvent) : this( + startTime = event.eventTime ?: LocalDateTime.now(), + projectId = event.projectId, + pipelineId = event.pipelineId, + buildId = event.buildId, + jobId = event.jobId, + stepId = event.stepId, + status = checkNotNull(event.buildStatus), + atomCode = event.atomCode, + eventType = checkNotNull(event.toEventType()), + endTime = null + ) + + companion object { + const val DELIMITER = "," + fun load(str: String?): MetricsUserPO? { + if (str.isNullOrBlank()) return null + val list = str.split(DELIMITER) + if (list.size != 10) return null + return MetricsUserPO( + LocalDateTime.ofInstant(Instant.ofEpochSecond(list[0].toLong()), ZoneOffset.ofHours(8)), + list[1], + list[2], + list[3], + list[4].ifEmpty { null }, + list[5].ifEmpty { null }, + list[6], + list[7].ifEmpty { null }, + CallBackEvent.valueOf(list[8]), + list[9].ifEmpty { null }?.let { + LocalDateTime.ofInstant(Instant.ofEpochSecond(it.toLong()), ZoneOffset.ofHours(8)) + } + ) + } + } + + override fun toString(): String { + return startTime.toInstant(ZoneOffset.ofHours(8)).epochSecond.toString() + DELIMITER + + projectId + DELIMITER + + pipelineId + DELIMITER + + buildId + DELIMITER + + (jobId ?: "") + DELIMITER + + (stepId ?: "") + DELIMITER + + status + DELIMITER + + (atomCode ?: "") + DELIMITER + + eventType.name + DELIMITER + + (endTime?.toInstant(ZoneOffset.ofHours(8))?.epochSecond?.toString() ?: "") + } +} diff --git a/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/config/MetricsUserConfig.kt b/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/config/MetricsUserConfig.kt new file mode 100644 index 00000000000..99253eb5516 --- /dev/null +++ b/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/config/MetricsUserConfig.kt @@ -0,0 +1,97 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.metrics.config + +import io.micrometer.core.instrument.Clock +import io.micrometer.prometheus.PrometheusConfig +import io.micrometer.prometheus.PrometheusMeterRegistry +import io.prometheus.client.CollectorRegistry +import io.prometheus.client.exporter.common.TextFormat +import org.springframework.beans.factory.annotation.Value +import org.springframework.boot.actuate.endpoint.annotation.ReadOperation +import org.springframework.boot.actuate.endpoint.web.annotation.WebEndpoint +import org.springframework.boot.actuate.metrics.export.prometheus.TextOutputFormat +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration + +@Configuration +@ConditionalOnProperty(name = ["metrics.user.enable"], havingValue = "true", matchIfMissing = false) +class MetricsUserConfig { + + companion object { + const val gaugeBuildKey = "pipeline_running_time_seconds" + const val gaugeBuildStatusKey = "pipeline_status_info" + const val gaugeBuildJobKey = "pipeline_job_running_time_seconds" + const val gaugeBuildStepKey = "pipeline_step_running_time_seconds" + const val gaugeBuildStepStatusKey = "pipeline_step_status_info" + } + + @Value("\${metrics.user.localCacheMaxSize:100000}") + val localCacheMaxSize: Long = 100000L + + @Value("\${metrics.user.enable:false}") + val metricsUserEnabled: Boolean = false + + /*注册默认的 prometheusMeterRegistry*/ + @Bean + fun prometheusMeterRegistry( + prometheusConfig: PrometheusConfig, + collectorRegistry: CollectorRegistry, + clock: Clock + ): PrometheusMeterRegistry { + return PrometheusMeterRegistry(prometheusConfig, collectorRegistry, clock) + } + + @Bean + fun userPrometheusMeterRegistry(): PrometheusMeterRegistry { + return PrometheusMeterRegistry(PrometheusConfig.DEFAULT) + } + + @Bean + fun userPrometheusEndpoint(userPrometheusMeterRegistry: PrometheusMeterRegistry): UserPrometheusEndpoint { + return UserPrometheusEndpoint(userPrometheusMeterRegistry) + } + + @WebEndpoint(id = "userPrometheus") + class UserPrometheusEndpoint(private val meterRegistry: PrometheusMeterRegistry) { + + @ReadOperation(producesFrom = TextOutputFormat::class) + fun scrape(): String { + return meterRegistry.scrape( + TextFormat.CONTENT_TYPE_004, setOf( + gaugeBuildKey, + gaugeBuildStatusKey, + gaugeBuildJobKey, + gaugeBuildStepKey, + gaugeBuildStepStatusKey + ) + ) + } + } +} diff --git a/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/config/MetricsUserListenerConfiguration.kt b/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/config/MetricsUserListenerConfiguration.kt new file mode 100644 index 00000000000..4b4f3b102f9 --- /dev/null +++ b/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/config/MetricsUserListenerConfiguration.kt @@ -0,0 +1,93 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.metrics.config + +import com.tencent.devops.common.event.dispatcher.pipeline.mq.MQ +import com.tencent.devops.common.event.dispatcher.pipeline.mq.Tools +import com.tencent.devops.metrics.listener.BuildMetricsUserListener +import org.springframework.amqp.core.Binding +import org.springframework.amqp.core.BindingBuilder +import org.springframework.amqp.core.FanoutExchange +import org.springframework.amqp.core.Queue +import org.springframework.amqp.rabbit.connection.ConnectionFactory +import org.springframework.amqp.rabbit.core.RabbitAdmin +import org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer +import org.springframework.amqp.support.converter.Jackson2JsonMessageConverter +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration + +@Configuration +@ConditionalOnProperty(name = ["metrics.user.enable"], havingValue = "true", matchIfMissing = false) +class MetricsUserListenerConfiguration { + /** + * 构建构建回调广播交换机 + */ + @Bean + fun pipelineBuildStatusCallbackFanoutExchange(): FanoutExchange { + val fanoutExchange = FanoutExchange(MQ.EXCHANGE_PIPELINE_BUILD_CALL_BACK_FANOUT, true, false) + fanoutExchange.isDelayed = true + return fanoutExchange + } + + @Bean + fun pipelineBuildStatusMetricsQueue(): Queue { + return Queue(MQ.QUEUE_PIPELINE_BUILD_STATUS_METRICS) + } + + @Bean + fun pipelineBuildStatusMetricsQueueBind( + @Autowired pipelineBuildStatusMetricsQueue: Queue, + @Autowired pipelineBuildStatusCallbackFanoutExchange: FanoutExchange + ): Binding { + return BindingBuilder.bind(pipelineBuildStatusMetricsQueue).to(pipelineBuildStatusCallbackFanoutExchange) + } + + @Bean + fun pipelineBuildCallBackListenerContainer( + @Autowired connectionFactory: ConnectionFactory, + @Autowired pipelineBuildStatusMetricsQueue: Queue, + @Autowired rabbitAdmin: RabbitAdmin, + @Autowired buildListener: BuildMetricsUserListener, + @Autowired messageConverter: Jackson2JsonMessageConverter + ): SimpleMessageListenerContainer { + + return Tools.createSimpleMessageListenerContainer( + connectionFactory = connectionFactory, + queue = pipelineBuildStatusMetricsQueue, + rabbitAdmin = rabbitAdmin, + buildListener = buildListener, + messageConverter = messageConverter, + startConsumerMinInterval = 10000, + consecutiveActiveTrigger = 5, + concurrency = 1, + maxConcurrency = 50 + ) + } +} diff --git a/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/listener/BuildMetricsUserListener.kt b/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/listener/BuildMetricsUserListener.kt new file mode 100644 index 00000000000..20c25687d5b --- /dev/null +++ b/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/listener/BuildMetricsUserListener.kt @@ -0,0 +1,53 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.metrics.listener + +import com.tencent.devops.common.event.listener.Listener +import com.tencent.devops.common.event.pojo.pipeline.PipelineBuildStatusBroadCastEvent +import com.tencent.devops.metrics.service.builds.MetricsUserService +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty +import org.springframework.stereotype.Component + +@Component +@ConditionalOnProperty(name = ["metrics.user.enable"], havingValue = "true", matchIfMissing = false) +class BuildMetricsUserListener @Autowired constructor( + private val metricsUserService: MetricsUserService +) : Listener { + + companion object { + val logger = LoggerFactory.getLogger(BuildMetricsUserListener::class.java) + } + + override fun execute(event: PipelineBuildStatusBroadCastEvent) { + kotlin.runCatching { metricsUserService.execute(event) }.onFailure { + logger.warn("BuildMetricsUserListener error |$event", it) + } + } +} diff --git a/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/service/CacheProjectInfoService.kt b/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/service/CacheProjectInfoService.kt index ee7e448cc13..dccbeda6eb9 100644 --- a/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/service/CacheProjectInfoService.kt +++ b/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/service/CacheProjectInfoService.kt @@ -31,6 +31,7 @@ package com.tencent.devops.metrics.service import com.github.benmanes.caffeine.cache.Caffeine import com.tencent.devops.common.client.Client import com.tencent.devops.project.api.service.ServiceProjectResource +import com.tencent.devops.project.pojo.ProjectVO import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service import java.util.concurrent.TimeUnit @@ -39,16 +40,16 @@ import java.util.concurrent.TimeUnit class CacheProjectInfoService @Autowired constructor( private val client: Client ) { - private val productIdCache = Caffeine.newBuilder() + private val projectVOCache = Caffeine.newBuilder() .maximumSize(10000) - .expireAfterWrite(1, TimeUnit.DAYS) - .build() + .expireAfterWrite(1, TimeUnit.HOURS) + .build() - fun getProjectId(projectId: String): Int { - return productIdCache.getIfPresent(projectId) ?: run { - val productId = client.get(ServiceProjectResource::class).get(projectId).data?.productId ?: 0 - productIdCache.put(projectId, productId) - productId + fun getProject(projectId: String): ProjectVO? { + return projectVOCache.getIfPresent(projectId) ?: run { + val projectVO = client.get(ServiceProjectResource::class).get(projectId).data ?: return null + projectVOCache.put(projectId, projectVO) + projectVO } } } diff --git a/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/service/builds/MetricsCacheService.kt b/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/service/builds/MetricsCacheService.kt new file mode 100644 index 00000000000..7f0f6075241 --- /dev/null +++ b/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/service/builds/MetricsCacheService.kt @@ -0,0 +1,310 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + */ + +package com.tencent.devops.metrics.service.builds + +import com.google.common.collect.MapMaker +import com.google.common.hash.Hashing +import com.tencent.devops.common.redis.RedisOperation +import com.tencent.devops.metrics.config.MetricsUserConfig +import com.tencent.devops.metrics.pojo.po.MetricsUserPO +import groovy.util.ObservableMap +import java.beans.PropertyChangeEvent +import java.time.LocalDateTime +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.beans.factory.annotation.Qualifier +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty +import org.springframework.stereotype.Service + +@Service +@ConditionalOnProperty(name = ["metrics.user.enable"], havingValue = "true", matchIfMissing = false) +class MetricsCacheService @Autowired constructor( + private val metricsHeartBeatService: MetricsHeartBeatService, + @Qualifier("redisStringHashOperation") + private val redisHashOperation: RedisOperation, + private val metricsUserConfig: MetricsUserConfig +) { + + private val cache = ObservableMap( + MapMaker() + .concurrencyLevel(10) + .makeMap() + ) + + lateinit var addFunction: (key: String, value: MetricsUserPO) -> Unit + + lateinit var removeFunction: (key: String, value: MetricsUserPO) -> Unit + + lateinit var updateFunction: (key: String, oldValue: MetricsUserPO, newValue: MetricsUserPO) -> Unit + + companion object { + val logger: Logger = LoggerFactory.getLogger(MetricsCacheService::class.java) + } + + fun init(checkStatusSet: MutableSet) { + cache.addPropertyChangeListener { change: PropertyChangeEvent -> + when { + change is ObservableMap.PropertyAddedEvent && this::addFunction.isInitialized -> { + addFunction(change.propertyName, change.newValue as MetricsUserPO) + } + + change is ObservableMap.PropertyUpdatedEvent && this::updateFunction.isInitialized -> { + updateFunction( + change.propertyName, + change.oldValue as MetricsUserPO, + change.newValue as MetricsUserPO + ) + } + + change is ObservableMap.PropertyRemovedEvent && this::removeFunction.isInitialized -> { + removeFunction(change.propertyName, change.oldValue as MetricsUserPO) + } + } + } + Thread( + CacheUpdateProcess( + updateKey = metricsHeartBeatService.updateKey(), + podKey = metricsHeartBeatService::podKey, + podHashKey = metricsHeartBeatService.getPodName(), + cache = cache, + checkStatusSet = checkStatusSet, + redisOperation = redisHashOperation + ) + ).start() + metricsHeartBeatService.init() + } + + fun removeCache(key: String) { + cache.remove(key) + redisHashOperation.hdelete(metricsHeartBeatService.podKey(metricsHeartBeatService.getPodName()), key) + } + + fun buildCacheStart( + buildId: String, + executeCount: Int, + data: MetricsUserPO + ): String { + val key = hash("$buildId-$executeCount") + cacheStart(key, data) + return key + } + + fun buildCacheEnd( + buildId: String, + executeCount: Int, + data: MetricsUserPO + ): String { + val key = hash("$buildId-$executeCount") + cacheEnd(key, data) + return key + } + + fun jobCacheStart( + buildId: String, + jobId: String, + executeCount: Int, + data: MetricsUserPO + ): String { + val key = hash("$buildId-$jobId-$executeCount") + cacheStart(key, data) + return key + } + + fun jobCacheEnd( + buildId: String, + jobId: String, + executeCount: Int, + data: MetricsUserPO + ): String { + val key = hash("$buildId-$jobId-$executeCount") + cacheEnd(key, data) + return key + } + + fun stepCacheStart( + buildId: String, + stepId: String, + executeCount: Int, + data: MetricsUserPO + ): String { + val key = hash("$buildId-$stepId-$executeCount") + cacheStart(key, data) + return key + } + + fun stepCacheEnd( + buildId: String, + stepId: String, + executeCount: Int, + data: MetricsUserPO + ): String { + val key = hash("$buildId-$stepId-$executeCount") + cacheEnd(key, data) + return key + } + + private fun hash(input: String): String { + return Hashing.murmur3_128().hashBytes(input.toByteArray()).toString() + } + + private fun cacheStart(key: String, data: MetricsUserPO) { + /*如果本地缓存已经达到上限，则推到redis来进行分发*/ + if (cache.size > metricsUserConfig.localCacheMaxSize) { + logger.warn("METRICS_USER_WARN_LOG|local cache size exceeds maximum.") + // 避免buffer过大，超过2倍maxLocalCacheSize 丢弃多余数据 + if (redisHashOperation.hsize(metricsHeartBeatService.bufferKey()) > + 2 * metricsUserConfig.localCacheMaxSize + ) { + logger.error("METRICS_USER_WARN_LOG|buffer is full.") + return + } + redisHashOperation.hset(metricsHeartBeatService.bufferKey(), key, data.toString()) + return + } + redisHashOperation.hset( + key = metricsHeartBeatService.podKey(metricsHeartBeatService.getPodName()), + hashKey = key, + values = data.toString() + ) + cache[key] = data + } + + private fun cacheEnd(key: String, data: MetricsUserPO) { + val cacheKey = cache[key] as MetricsUserPO? + if (cacheKey != null) { + cache[key] = data.apply { startTime = cacheKey.startTime } + redisHashOperation.hset( + key = metricsHeartBeatService.podKey(metricsHeartBeatService.getPodName()), + hashKey = key, + values = data.toString() + ) + } else { + redisHashOperation.hset(metricsHeartBeatService.updateKey(), key, data.toString()) + } + } + + private class CacheUpdateProcess( + private val updateKey: String, + private val podKey: (String) -> String, + private val podHashKey: String, + private val cache: ObservableMap, + private val checkStatusSet: MutableSet, + private val redisOperation: RedisOperation + ) : Runnable { + + companion object { + const val SLEEP = 10000L + } + + override fun run() { + logger.info("CacheUpdateProcess begin") + while (true) { + kotlin.runCatching { executeUpdate() }.onFailure { + logger.error("metrics execute update process find a error|${it.message}", it) + } + kotlin.runCatching { executeAdd() }.onFailure { + logger.error("metrics execute add process find a error|${it.message}", it) + } + kotlin.runCatching { executeCheck() }.onFailure { + logger.error("metrics execute check process find a error|${it.message}", it) + } + Thread.sleep(SLEEP) + } + } + + /** + * 执行更新操作。 + * + * 该方法会从Redis中获取更新数据的键列表，并根据键的存在与否进行相应的操作。 + * 如果键存在于缓存中，则表示状态维护在当前实例，将更新数据应用到缓存中，并从Redis中删除该键。 + * + * @return 无 + */ + private fun executeUpdate() { + val update = redisOperation.hkeys(updateKey) ?: return + val snapshot = cache.keys.toList() + update.parallelStream().forEach { ready -> + if (ready in snapshot) { + // 如果key存在，说明状态维护在当前实例 + val load = MetricsUserPO.load(redisOperation.hget(updateKey, ready)) ?: return@forEach + redisOperation.hdelete(updateKey, ready) + cache[ready] = load.apply { startTime = (cache[ready] as MetricsUserPO?)?.startTime ?: startTime } + } + } + } + + /** + * 执行新增操作。 + * + * 该方法会从Redis中获取新增数据的键列表，并根据键的存在与否进行相应的操作。 + * 如果键不存在于缓存中，则表示需要新增该数据，将其添加到缓存中。 + * + * 同时，方法会遍历缓存中的键列表，如果某个键不在新增数据的键列表中，则表示该数据已失效，将其从缓存中移除。 + * + * @return 无 + */ + private fun executeAdd() { + val add = redisOperation.hkeys(podKey(podHashKey)) ?: return + val snapshot = cache.keys.toList() + logger.info("executeAdd local size=${snapshot.size}|${add.size}") + add.parallelStream().forEach { ready -> + if (ready !in snapshot) { + // 如果key不存在，说明是需要新增的 + val load = MetricsUserPO.load(redisOperation.hget(podKey(podHashKey), ready)) ?: return@forEach + cache[ready] = load + } + } + snapshot.parallelStream().forEach { already -> + if (already !in add) { + cache.remove(already) + } + } + } + + /** + * 执行检查操作。 + * + * 该方法会遍历缓存中的数据，对于运行时间超过一个小时的数据，将其加入检查队列。 + * + * @return 无 + */ + private fun executeCheck() { + /* 运行超过一个小时的，加入检查队列 */ + val limit = LocalDateTime.now().plusHours(-1) + val snapshot = cache.keys.toList() + snapshot.parallelStream().forEach { key -> + val value = cache[key] as MetricsUserPO? ?: return@forEach + if (value.buildId !in checkStatusSet && value.startTime < limit) { + checkStatusSet.add(value.buildId) + } + } + } + } +} diff --git a/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/service/builds/MetricsHeartBeatService.kt b/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/service/builds/MetricsHeartBeatService.kt new file mode 100644 index 00000000000..fdbd8b733cc --- /dev/null +++ b/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/service/builds/MetricsHeartBeatService.kt @@ -0,0 +1,295 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + */ + +package com.tencent.devops.metrics.service.builds + +import com.tencent.devops.common.redis.RedisLock +import com.tencent.devops.common.redis.RedisOperation +import com.tencent.devops.common.service.BkTag +import com.tencent.devops.metrics.config.MetricsUserConfig +import com.tencent.devops.metrics.pojo.po.MetricsUserPO +import java.time.LocalDateTime +import java.time.ZoneOffset +import java.util.UUID +import org.apache.commons.lang3.math.NumberUtils +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.beans.factory.annotation.Qualifier +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty +import org.springframework.stereotype.Service + +@Service +@ConditionalOnProperty(name = ["metrics.user.enable"], havingValue = "true", matchIfMissing = false) +class MetricsHeartBeatService @Autowired constructor( + @Qualifier("redisStringHashOperation") + private val redisHashOperation: RedisOperation, + private val metricsUserConfig: MetricsUserConfig, + private val bkTag: BkTag +) { + private val podName: String = System.getenv("POD_NAME") ?: UUID.randomUUID().toString() + + companion object { + private val logger = LoggerFactory.getLogger(MetricsHeartBeatService::class.java) + } + + fun heartBeatKey() = "build_metrics:${bkTag.getLocalTag()}:heart_beat" + fun podKey(key: String) = "build_metrics:${bkTag.getLocalTag()}:pod:$key" + fun updateKey() = "build_metrics:${bkTag.getLocalTag()}:update" + fun bufferKey() = "build_metrics:${bkTag.getLocalTag()}:buffer" + fun redisLockKey() = "metrics_heart_beat_manager_process_redis_lock_${bkTag.getLocalTag()}" + + fun init() { + Thread(HeartBeatProcess(heartBeatKey(), podName, redisHashOperation)).start() + Thread( + HeartBeatManagerProcess( + redisLockKey = redisLockKey(), + updateKey = updateKey(), + bufferKey = bufferKey(), + heartBeatKey = heartBeatKey(), + podKey = ::podKey, + redisOperation = redisHashOperation, + maxLocalCacheSize = metricsUserConfig.localCacheMaxSize + ) + ).start() + } + + fun getPodName(): String = podName + + private class HeartBeatProcess( + private val heartBeatKey: String, + private val podHashKey: String, + private val redisOperation: RedisOperation + ) : Runnable { + + companion object { + const val SLEEP = 5000L + } + + override fun run() { + logger.info("HeartBeatProcess run") + while (true) { + kotlin.runCatching { execute() }.onFailure { + logger.error("metrics heart beat error|${it.message}", it) + } + Thread.sleep(SLEEP) + } + } + + private fun execute() { + redisOperation.hset( + heartBeatKey, + podHashKey, + LocalDateTime.now().toInstant(ZoneOffset.ofHours(8)).epochSecond.toString() + ) + } + } + + private class HeartBeatManagerProcess( + private val redisLockKey: String, + private val updateKey: String, + private val bufferKey: String, + private val heartBeatKey: String, + private val podKey: (String) -> String, + private val redisOperation: RedisOperation, + private val maxLocalCacheSize: Long + ) : Runnable { + + companion object { + const val SLEEP = 15000L + const val CHUNKED = 100 + } + + override fun run() { + logger.info("HeartBeatManagerProcess begin") + while (true) { + val redisLock = RedisLock(redisOperation, redisLockKey, 60L) + try { + val lockSuccess = redisLock.tryLock() + if (lockSuccess) { + logger.info("HeartBeatManagerProcess get lock.") + heartBeatCheck() + invalidUpdateCheck() + bufferCheck() + } + } catch (e: Throwable) { + logger.error("HeartBeatManagerProcess failed ${e.message}", e) + } finally { + Thread.sleep(SLEEP) + redisLock.unlock() + } + } + } + + /** + * 检查失效的更新数据并进行清理。 + * + * 该方法会从Redis中获取更新数据的键列表，并根据更新数据的结束时间判断是否失效。 + * 如果更新数据的结束时间早于1小时前，则将其标记为失效数据。 + * + * 失效的更新数据将被删除。 + * + * @return 无 + */ + private fun invalidUpdateCheck() { + val updateKeys = redisOperation.hkeys(updateKey)?.ifEmpty { null } ?: return + val limit = LocalDateTime.now().plusHours(-1) + val needDelete = mutableListOf() + updateKeys.chunked(CHUNKED).forEach { keys -> + val updateValues = redisOperation.hmGet(updateKey, keys) + ?: return@forEach + keys.forEachIndexed { index, key -> + val load = MetricsUserPO.load(updateValues[index]) ?: return@forEachIndexed + if (load.endTime!! < limit) { + needDelete.add(key) + } + } + } + if (needDelete.isNotEmpty()) { + redisOperation.hdelete(updateKey, needDelete.toTypedArray()) + } + } + + /** + * 检查缓冲区并将指标数据分配到可用的Pod。 + * + * 判断是否需要检测buffer，如果buffer大小为0，则直接返回。 + * 获取可用的Pod列表，如果列表为空，则直接返回。 + * 检测可用空间并计算每个Pod的可用空间大小。 + * 循环处理直到没有更多可用空间或者buffer大小小于一个单位。 + * 选择可用空间最大的宿主Pod。 + * 如果该Pod的可用空间减去一个单位的大小小于等于0，则表示该Pod已无法容纳更多指标，结束循环。 + * 获取一个单位的指标数据进行处理。 + * 将指标数据存储到目标Pod中。 + * 删除buffer中对应的数据。 + * 如果buffer大小小于一个单位，则结束循环。 + * 刷新可用空间。 + */ + private fun bufferCheck() { + // 判断是否需要检测buffer + val buffSize = redisOperation.hsize(bufferKey) + if (buffSize == 0L) return + // 获取可用pod + val livePods = redisOperation.hkeys(heartBeatKey) ?: return + // 检测可用空间 + val availablePodsSizeMap = livePods.associateWith { podKey -> + maxLocalCacheSize - redisOperation.hsize(podKey(podKey)) + }.toMutableMap() + while (true) { + // 选择最佳宿主Pod + val targetPod = availablePodsSizeMap.maxByOrNull { it.value } ?: break + if (targetPod.value - CHUNKED <= 0) { + // 目前已没有pod能容纳下更多指标了 + break + } + // 获取一个单位进行处理 + val cursor = redisOperation.hscan(bufferKey, count = CHUNKED.toLong()) + val loadedKeys = mutableListOf() + while (cursor.hasNext()) { + val keyValue = cursor.next() + redisOperation.hset(podKey(targetPod.key), keyValue.key, keyValue.value) + loadedKeys.add(keyValue.key) + } + // 删除buffer对应数据 + redisOperation.hdelete(bufferKey, loadedKeys.toTypedArray()) + // 再次检测buffer大小，如果小于一个单位则break + if (redisOperation.hsize(bufferKey) < CHUNKED) break + // 刷新可用空间 + availablePodsSizeMap[targetPod.key] = redisOperation.hsize(podKey(targetPod.key)) + } + } + + /** + * 检查心跳状态并处理失效的Pod。 + * + * 该方法会从Redis中获取心跳信息，并根据最后在线时间判断Pod的状态。 + * 如果Pod的最后在线时间早于1分钟前，则将其标记为失效Pod。 + * 如果Pod的最后在线时间在1分钟内，则将其标记为正常Pod。 + * + * 失效的Pod将被移除，并且与之相关的指标缓存也会被删除。 + * + * @return 无 + */ + private fun heartBeatCheck() { + val heartBeats = redisOperation.hentries(heartBeatKey) ?: return + val limit = LocalDateTime.now().plusMinutes(-1).toInstant(ZoneOffset.ofHours(8)).epochSecond + val lose = mutableListOf() + val live = mutableListOf() + heartBeats.toList().parallelStream().forEach { (podName, lastOnlineTime) -> + if (NumberUtils.toLong(lastOnlineTime) < limit) { + lose.add(podName) + } else { + live.add(podName) + } + } + logger.info("heartBeatCheck start check lose=$lose|live=$live") + if (live.isEmpty()) { + return + } + lose.forEach { losePod -> + if (afterLosePod(losePod, live)) { + redisOperation.hdelete(heartBeatKey, losePod) + redisOperation.delete(podKey(losePod)) + } + } + } + + /** + * 在失效Pod之后处理相关操作。 + * + * 该方法会根据失效Pod的名称和正常Pod的列表，将失效Pod的指标数据转移给正常Pod。 + * + * 首先，方法会从Redis中获取失效Pod的指标键列表。如果列表为空，则表示失效Pod已被处理，直接返回true。 + * + * 然后，方法会将失效Pod的指标数据按照近似负载均衡的方式分配给正常Pod。分配的方式是将失效Pod的指标键值对 + * 逐个转移到正常Pod中，并删除失效Pod中对应的键值对。 + * + * 最后，方法会再次校验失效Pod的指标键列表，如果为空，则表示数据转移成功，返回true；否则返回false。 + * + * @param losePod 失效Pod的名称 + * @param live 正常Pod的列表 + * @return 数据转移是否成功的布尔值。如果失效Pod的指标键列表为空，则返回true；否则返回false。 + */ + private fun afterLosePod(losePod: String, live: List): Boolean { + val losePodKeys = redisOperation.hkeys(podKey(losePod))?.ifEmpty { null } ?: return true + // 分块处理，优化性能。 + losePodKeys.chunked(CHUNKED).forEachIndexed { index, keys -> + val losePodValues = redisOperation.hmGet(podKey(losePod), keys) + ?: return@forEachIndexed + // 分配近似达到负载均衡的效果 + redisOperation.hmset( + podKey(live[index % live.size]), + keys.zip(losePodValues).toMap() + ) + redisOperation.hdelete(podKey(losePod), keys.toTypedArray()) + } + // 双重校验数据一致性 + val check = redisOperation.hkeys(podKey(losePod)) + return check.isNullOrEmpty() + } + } +} diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/service/RepoServiceFactory.kt b/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/service/builds/MetricsUserRunner.kt similarity index 58% rename from src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/service/RepoServiceFactory.kt rename to src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/service/builds/MetricsUserRunner.kt index bea24d3980c..e07176fe05c 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/service/RepoServiceFactory.kt +++ b/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/service/builds/MetricsUserRunner.kt @@ -23,35 +23,28 @@ * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * */ -package com.tencent.devops.worker.common.service - -import com.tencent.devops.common.api.util.PropertyUtil -import java.util.concurrent.ConcurrentHashMap - -object RepoServiceFactory { - - private val repoServiceMap = ConcurrentHashMap() +package com.tencent.devops.metrics.service.builds - private const val REPO_CLASS_NAME = "repo.class.name" +import com.tencent.devops.metrics.config.MetricsUserConfig +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.ApplicationArguments +import org.springframework.boot.ApplicationRunner +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty +import org.springframework.stereotype.Component - private const val AGENT_PROPERTIES_FILE_NAME = "/.agent.properties" +@Component +@ConditionalOnProperty(name = ["metrics.user.enable"], havingValue = "true", matchIfMissing = false) +class MetricsUserRunner @Autowired constructor( + private val metricsUserService: MetricsUserService, + private val metricsUserConfig: MetricsUserConfig +) : ApplicationRunner { - /** - * 根据配置文件的类名获取实现RepoService的对象 - * @return 实现RepoService的对象 - */ - fun getInstance(): RepoService { - // 从配置文件读取类名 - val className = PropertyUtil.getPropertyValue(REPO_CLASS_NAME, AGENT_PROPERTIES_FILE_NAME) - // 根据类名从缓存中获取实现RepoService的对象 - var repoService = repoServiceMap[className] - if (repoService == null) { - // 通过反射生成对象并放入缓存中 - repoService = Class.forName(className).newInstance() as RepoService - repoServiceMap[className] = repoService + override fun run(args: ApplicationArguments) { + if (metricsUserConfig.metricsUserEnabled) { + metricsUserService.init() } - return repoService } } diff --git a/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/service/builds/MetricsUserService.kt b/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/service/builds/MetricsUserService.kt new file mode 100644 index 00000000000..0ef3de632e9 --- /dev/null +++ b/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/service/builds/MetricsUserService.kt @@ -0,0 +1,509 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + */ + +package com.tencent.devops.metrics.service.builds + +import com.github.benmanes.caffeine.cache.Caffeine +import com.google.common.collect.MapMaker +import com.tencent.devops.common.client.Client +import com.tencent.devops.common.event.pojo.pipeline.PipelineBuildStatusBroadCastEvent +import com.tencent.devops.common.pipeline.event.CallBackEvent +import com.tencent.devops.metrics.config.MetricsUserConfig +import com.tencent.devops.metrics.pojo.po.MetricsLocalPO +import com.tencent.devops.metrics.pojo.po.MetricsUserPO +import com.tencent.devops.process.api.service.ServiceBuildResource +import com.tencent.devops.project.api.service.ServiceProjectResource +import io.micrometer.core.instrument.Gauge +import io.micrometer.core.instrument.Meter +import io.micrometer.prometheus.PrometheusMeterRegistry +import java.time.Duration +import java.time.LocalDateTime +import java.util.LinkedList +import java.util.concurrent.ConcurrentMap +import java.util.concurrent.TimeUnit +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.beans.factory.annotation.Qualifier +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty +import org.springframework.scheduling.annotation.Scheduled +import org.springframework.stereotype.Service + +@Service +@ConditionalOnProperty(name = ["metrics.user.enable"], havingValue = "true", matchIfMissing = false) +class MetricsUserService @Autowired constructor( + @Qualifier("userPrometheusMeterRegistry") + private val registry: PrometheusMeterRegistry, + private val metricsCacheService: MetricsCacheService, + private val metricsUserConfig: MetricsUserConfig, + private val client: Client +) { + private val local = MapMaker() + .concurrencyLevel(10) + .makeMap() + + /* 延迟删除队列 */ + val delayArray: LinkedList>> = + LinkedList(MutableList(DELAY_LIMIT) { mutableListOf() }) + + /* 疑似构建状态未同步队列,以buildId为单位 */ + val uncheckArray: MutableSet = mutableSetOf() + + private val buildMetricsCache = Caffeine.newBuilder() + .maximumSize(10000) + .expireAfterWrite(10, TimeUnit.MINUTES) + .build { key -> + kotlin.runCatching { + client.get(ServiceProjectResource::class).get( + englishName = key + ).data?.properties?.buildMetrics + }.getOrNull() ?: false + } + + /** + * 定时任务：检查构建状态。 + * + * 该方法会定时执行: 每10分钟运行一次 + * + * 方法首先生成一个未检查的构建ID列表的快照，并初始化一个待删除的构建ID列表。 + * 然后，将未检查的构建ID列表按照指定的块大小进行分块处理，每次处理一个块。 + * 对于每个块，方法会调用接口批量获取构建的基本信息，并获取返回结果中已完成的构建ID列表。 + * 将这些已完成的构建ID添加到待删除的构建ID列表中。 + * + * 接下来，方法会生成本地缓存的键列表，并遍历每个键。 + * 对于每个键，方法会获取对应的数据，并检查其构建ID是否在待删除的构建ID列表中。 + * 如果是，则从缓存中移除该键，并从未检查的构建ID列表中移除该构建ID。 + * + * @return 无 + */ + @Scheduled(cron = "0 0/10 * * * ?") + fun checkBuildStatusJob() { + logger.info("=========>> check build status job start <<=========") + // 生成快照 + val unchecks = uncheckArray.toList() + val ready2delete = mutableListOf() + unchecks.chunked(CHUNK_SIZE).forEach { chunk -> + val res = kotlin.runCatching { + client.get(ServiceBuildResource::class).batchServiceBasic( + buildIds = chunk.toSet() + ).data + }.getOrNull() ?: return@forEach + ready2delete.addAll(res.filter { it.value.status?.isFinish() == true }.map { it.key }) + } + + // 生成local快照 + val keys = local.keys.toList() + keys.forEach { key -> + val value = local[key] ?: return@forEach + if (value.data.buildId !in ready2delete) return@forEach + metricsCacheService.removeCache(key) + uncheckArray.remove(value.data.buildId) + } + } + + companion object { + val logger: Logger = LoggerFactory.getLogger(MetricsUserService::class.java) + const val DELAY_LIMIT = 5 + const val CHUNK_SIZE = 100 + } + + class DeleteDelayProcess( + private val delayArray: LinkedList>>, + private val registry: PrometheusMeterRegistry, + private val local: ConcurrentMap + ) : Runnable { + + companion object { + const val SLEEP = 60000L + } + + override fun run() { + while (true) { + kotlin.runCatching { execute() } + Thread.sleep(SLEEP) + } + } + + /** + * 延迟删除操作。 + * + * 该方法会从延迟数组中获取待执行的操作列表，并逐个执行。 + * 对于每个操作，方法会遍历其指标列表，并从注册表中移除对应的指标。 + * 同时，方法会从本地缓存中移除对应的键。 + * + * @return 无 + */ + private fun execute() { + delayArray.addFirst(mutableListOf()) + val ready = delayArray.removeLast() + ready.forEach { (key, metrics) -> + metrics.meters.forEach { meter -> + registry.remove(meter) + } + local.remove(key) + } + } + } + + fun init() { + metricsCacheService.addFunction = this::metricsAdd + metricsCacheService.removeFunction = this::metricsRemove + metricsCacheService.updateFunction = this::metricsUpdate + metricsCacheService.init(uncheckArray) + Thread(DeleteDelayProcess(delayArray, registry, local)).start() + } + + private fun check(event: PipelineBuildStatusBroadCastEvent): Boolean { + return buildMetricsCache.get(event.projectId) ?: false + } + + @Suppress("ComplexMethod") + fun execute(event: PipelineBuildStatusBroadCastEvent) { + if (!check(event) || !metricsUserConfig.metricsUserEnabled) return + val date = MetricsUserPO(event) + /*防止mq队列堆积导致的延迟信息进入处理，如果生产超过5分钟就丢弃*/ + if (date.startTime < LocalDateTime.now().plusMinutes(-5)) return + when (date.eventType) { + CallBackEvent.BUILD_START -> { + date.startTime = checkNotNull(event.eventTime) + metricsCacheService.buildCacheStart(event.buildId, checkNotNull(event.executeCount), date) + } + + CallBackEvent.BUILD_JOB_START -> { + if (event.jobId == null) { + // job id 用户没填写将不会上报指标 + return + } + date.startTime = checkNotNull(event.eventTime) + metricsCacheService.jobCacheStart( + event.buildId, + checkNotNull(event.jobId), + checkNotNull(event.executeCount), + date + ) + } + + CallBackEvent.BUILD_TASK_START -> { + date.startTime = checkNotNull(event.eventTime) + if (event.stepId == null) { + // stepId id 用户没填写将不会上报指标 + return + } + metricsCacheService.stepCacheStart( + event.buildId, + checkNotNull(event.stepId), + checkNotNull(event.executeCount), + date + ) + } + + CallBackEvent.BUILD_END -> { + date.endTime = checkNotNull(event.eventTime) + metricsCacheService.buildCacheEnd(event.buildId, checkNotNull(event.executeCount), date) + } + + CallBackEvent.BUILD_JOB_END -> { + if (event.jobId == null) { + // job id 用户没填写将不会上报指标 + return + } + date.endTime = checkNotNull(event.eventTime) + metricsCacheService.jobCacheEnd( + event.buildId, + checkNotNull(event.jobId), + checkNotNull(event.executeCount), + date + ) + } + + CallBackEvent.BUILD_TASK_END -> { + date.endTime = checkNotNull(event.eventTime) + if (event.stepId == null) { + // stepId id 用户没填写将不会上报指标 + return + } + metricsCacheService.stepCacheEnd( + event.buildId, + checkNotNull(event.stepId), + checkNotNull(event.executeCount), + date + ) + } + + else -> {} + } + } + + /* 请勿直接调用该方法 */ + private fun metricsAdd(key: String, value: MetricsUserPO) { + local[key] = MetricsLocalPO(value) + logger.debug("metricsAdd|key={}|value={}|localSize={}", key, value, local.size) + with(value) { + when (eventType) { + CallBackEvent.BUILD_START -> { + val buildGauge = registerBuildGauge( + key = key, + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + description = "build metrics for $buildId" + ) + local[key]?.meters?.add(buildGauge) + val buildStatusGauge = registerBuildStatusGauge( + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + status = status, + description = "build status metrics for $buildId" + ) + local[key]?.meters?.add(buildStatusGauge) + } + + CallBackEvent.BUILD_JOB_START -> { + val buildJobGauge = registerBuildJobGauge( + key = key, + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + jobId = checkNotNull(jobId), + description = "job metrics for $buildId|$jobId" + ) + local[key]?.meters?.add(buildJobGauge) + } + + CallBackEvent.BUILD_TASK_START -> { + val buildStepGauge = registerBuildStepGauge( + key = key, + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + jobId = checkNotNull(jobId), + stepId = checkNotNull(stepId), + atomCode = checkNotNull(atomCode), + description = "step metrics for $buildId|$stepId" + ) + local[key]?.meters?.add(buildStepGauge) + val buildStepStatusGauge = registerBuildStepStatusGauge( + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + jobId = jobId!!, + stepId = stepId!!, + status = status, + description = "step status metrics for $buildId|$stepId" + ) + local[key]?.meters?.add(buildStepStatusGauge) + } + + else -> { + /*其余情况属于END状态，应当去除*/ + metricsCacheService.removeCache(key) + } + } + } + } + + /* 请勿直接调用该方法 */ + private fun metricsRemove(key: String, value: MetricsUserPO) { + val metrics = local[key] + logger.debug("metricsRemove|key={}|value={}|metrics={}", key, value, metrics) + if (metrics != null) { + // 异步删除 + delayArray.first.add(key to metrics) + } + } + + @Suppress("NestedBlockDepth") + /* 请勿直接调用该方法 */ + private fun metricsUpdate(key: String, oldValue: MetricsUserPO, newValue: MetricsUserPO) { + val metrics = local[key] + logger.debug("metricsUpdate|key={}|oldValue={}|newValue={}|metrics={}", key, oldValue, newValue, metrics) + if (metrics != null) { + metrics.data = newValue + with(newValue) { + when (eventType) { + CallBackEvent.BUILD_END -> { + metrics.meters.find { it.id.name == MetricsUserConfig.gaugeBuildStatusKey }?.run { + registry.remove(this) + } + metrics.meters.add( + registerBuildStatusGauge( + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + status = status, + description = "build status metrics for $buildId" + ) + ) + metricsCacheService.removeCache(key) + } + + CallBackEvent.BUILD_JOB_END -> { + metricsCacheService.removeCache(key) + } + + CallBackEvent.BUILD_TASK_END -> { + metrics.meters.find { it.id.name == MetricsUserConfig.gaugeBuildStepStatusKey }?.run { + registry.remove(this) + } + metrics.meters.add( + registerBuildStepStatusGauge( + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + jobId = jobId!!, + stepId = stepId!!, + status = status, + description = "step status metrics for $buildId|$stepId" + ) + ) + metricsCacheService.removeCache(key) + } + + else -> {} + } + } + } + } + + private fun registerBuildGauge( + key: String, + projectId: String, + pipelineId: String, + buildId: String, + description: String + ): Meter { + return Gauge.builder( + MetricsUserConfig.gaugeBuildKey, + local + ) { cache -> cache[key]?.let { computeStartTime(it) } ?: 0.0 } + .tags( + "projectId", projectId, + "pipeline_id", pipelineId, + "build_id", buildId + ) + .description(description) + .register(registry) + } + + private fun registerBuildStatusGauge( + projectId: String, + pipelineId: String, + buildId: String, + status: String, + description: String + ): Meter { + return Gauge.builder( + MetricsUserConfig.gaugeBuildStatusKey + ) { 1 } + .tags( + "projectId", projectId, + "pipeline_id", pipelineId, + "build_id", buildId, + "status", status + ) + .description(description) + .register(registry) + } + + private fun registerBuildJobGauge( + key: String, + projectId: String, + pipelineId: String, + buildId: String, + jobId: String, + description: String + ): Meter { + return Gauge.builder( + MetricsUserConfig.gaugeBuildJobKey, + local + ) { cache -> cache[key]?.let { computeStartTime(it) } ?: 0.0 } + .tags( + "projectId", projectId, + "pipeline_id", pipelineId, + "build_id", buildId, + "job_id", jobId + ) + .description(description) + .register(registry) + } + + private fun registerBuildStepGauge( + key: String, + projectId: String, + pipelineId: String, + buildId: String, + jobId: String, + stepId: String, + atomCode: String, + description: String + ): Meter { + return Gauge.builder( + MetricsUserConfig.gaugeBuildStepKey, + local + ) { cache -> cache[key]?.let { computeStartTime(it) } ?: 0.0 } + .tags( + "projectId", projectId, + "pipeline_id", pipelineId, + "build_id", buildId, + "job_id", jobId, + "step_id", stepId, + "plugin_id", atomCode + ) + .description(description) + .register(registry) + } + + private fun registerBuildStepStatusGauge( + projectId: String, + pipelineId: String, + buildId: String, + jobId: String, + stepId: String, + status: String, + description: String + ): Meter { + return Gauge.builder( + MetricsUserConfig.gaugeBuildStepStatusKey + ) { 1 } + .tags( + "projectId", projectId, + "pipeline_id", pipelineId, + "build_id", buildId, + "job_id", jobId, + "step_id", stepId, + "status", status + ) + .description(description) + .register(registry) + } + + private fun computeStartTime(cache: MetricsLocalPO): Double { + return Duration.between(cache.data.startTime, cache.data.endTime ?: LocalDateTime.now()).seconds.toDouble() + } +} diff --git a/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/service/impl/ProjectBuildSummaryServiceImpl.kt b/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/service/impl/ProjectBuildSummaryServiceImpl.kt index 60cfffc38a6..333b4324328 100644 --- a/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/service/impl/ProjectBuildSummaryServiceImpl.kt +++ b/src/backend/ci/core/metrics/biz-metrics/src/main/kotlin/com/tencent/devops/metrics/service/impl/ProjectBuildSummaryServiceImpl.kt @@ -37,6 +37,7 @@ import com.tencent.devops.metrics.service.CacheProjectInfoService import com.tencent.devops.metrics.service.ProjectBuildSummaryService import org.jooq.DSLContext import org.jooq.impl.DSL +import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service import java.time.LocalDate @@ -50,6 +51,7 @@ class ProjectBuildSummaryServiceImpl @Autowired constructor( ) : ProjectBuildSummaryService { companion object { + private val logger = LoggerFactory.getLogger(ProjectBuildSummaryServiceImpl::class.java) private fun projectBuildKey(key: String) = "ProjectBuild:$key" } @@ -63,11 +65,15 @@ class ProjectBuildSummaryServiceImpl @Autowired constructor( val lock = RedisLock(redisOperation, projectBuildKey(projectId), 120) lock.use { lock.lock() - val productId = cacheProjectInfoService.getProjectId(projectId) + val projectVO = cacheProjectInfoService.getProject(projectId) + if (projectVO?.enabled == false) { + logger.info("Project [${projectVO.englishName}] has disabled, skip build count") + return + } projectBuildSummaryDao.saveBuildCount( dslContext = dslContext, projectId = projectId, - productId = productId, + productId = projectVO?.productId ?: 0, trigger = trigger ) } @@ -81,7 +87,11 @@ class ProjectBuildSummaryServiceImpl @Autowired constructor( val lock = RedisLock(redisOperation, projectBuildKey(projectId), 120) lock.use { lock.lock() - val productId = cacheProjectInfoService.getProjectId(projectId) + val projectVO = cacheProjectInfoService.getProject(projectId) + if (projectVO?.enabled == false) { + logger.info("Project [${projectVO.englishName}] has disabled, skip user count") + return + } dslContext.transaction { configuration -> val transactionContext = DSL.using(configuration) val insert = projectBuildSummaryDao.saveProjectUser( @@ -93,7 +103,7 @@ class ProjectBuildSummaryServiceImpl @Autowired constructor( projectBuildSummaryDao.saveUserCount( dslContext = dslContext, projectId = projectId, - productId = productId, + productId = projectVO?.productId ?: 0, theDate = theDate ) } diff --git a/src/backend/ci/core/misc/biz-image/build.gradle.kts b/src/backend/ci/core/misc/biz-image/build.gradle.kts index 374abc7b040..38a93609e06 100644 --- a/src/backend/ci/core/misc/biz-image/build.gradle.kts +++ b/src/backend/ci/core/misc/biz-image/build.gradle.kts @@ -36,6 +36,7 @@ dependencies { api(project(":core:common:common-web")) api(project(":core:common:common-auth:common-auth-api")) api(project(":core:log:api-log")) + api(project(":core:dispatch:api-dispatch-kubernetes")) api("org.apache.commons:commons-compress") api("com.github.docker-java:docker-java") api("org.glassfish.jersey.core:jersey-client") diff --git a/src/backend/ci/core/misc/biz-image/src/main/kotlin/com/tencent/devops/image/service/InspectImageService.kt b/src/backend/ci/core/misc/biz-image/src/main/kotlin/com/tencent/devops/image/service/InspectImageService.kt index 6772dc29788..6641e759f0e 100644 --- a/src/backend/ci/core/misc/biz-image/src/main/kotlin/com/tencent/devops/image/service/InspectImageService.kt +++ b/src/backend/ci/core/misc/biz-image/src/main/kotlin/com/tencent/devops/image/service/InspectImageService.kt @@ -30,17 +30,13 @@ package com.tencent.devops.image.service import com.fasterxml.jackson.core.type.TypeReference import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue -import com.github.dockerjava.api.model.PullResponseItem -import com.github.dockerjava.core.DefaultDockerClientConfig -import com.github.dockerjava.core.DockerClientBuilder -import com.github.dockerjava.core.command.PullImageResultCallback import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID import com.tencent.devops.common.api.util.JsonUtil import com.tencent.devops.common.api.util.OkhttpUtils -import com.tencent.devops.image.config.DockerConfig +import com.tencent.devops.common.client.Client +import com.tencent.devops.dispatch.kubernetes.api.service.ServiceDockerImageResource import com.tencent.devops.image.pojo.CheckDockerImageRequest import com.tencent.devops.image.pojo.CheckDockerImageResponse -import com.tencent.devops.image.utils.CommonUtils import okhttp3.Headers.Companion.toHeaders import okhttp3.MediaType.Companion.toMediaTypeOrNull import okhttp3.Request @@ -49,23 +45,17 @@ import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.beans.factory.annotation.Value import org.springframework.stereotype.Service +import java.util.stream.Collectors @Service class InspectImageService @Autowired constructor( - dockerConfig: DockerConfig + private val client: Client ) { companion object { private val logger = LoggerFactory.getLogger(InspectImageService::class.java) } - private val config = DefaultDockerClientConfig.createDefaultConfigBuilder() - .withDockerConfig(dockerConfig.dockerConfig) - .withApiVersion(dockerConfig.apiVersion) - .build() - - private val dockerCli = DockerClientBuilder.getInstance(config).build() - @Value("\${image.checkImageUrl:}") var checkImageUrl: String? = null @@ -77,75 +67,9 @@ class InspectImageService @Autowired constructor( if (!checkImageUrl.isNullOrBlank()) { return checkRemoteDockerImage(userId, checkDockerImageRequestList) + } else { + return checkKubernetesDockerImage(userId, checkDockerImageRequestList) } - - val imageInspectList = mutableListOf() - checkDockerImageRequestList.parallelStream().forEach { - // 判断用户录入的镜像信息是否能正常拉取到镜像 - val imageName = it.imageName - try { - val authConfig = CommonUtils.getAuthConfig( - imageName = imageName, - registryHost = it.registryHost, - registryUser = it.registryUser, - registryPwd = it.registryPwd - ) - logger.info("Start pulling the image, image name：$imageName") - dockerCli.pullImageCmd(imageName).withAuthConfig(authConfig) - .exec(MyPullImageResultCallback(userId)).awaitCompletion() - logger.info("The image was pulled successfully. Image name：$imageName") - } catch (t: Throwable) { - logger.warn("Fail to pull the image $imageName of userId $userId", t) - imageInspectList.add( - CheckDockerImageResponse( - errorCode = -1, - errorMessage = t.message, - arch = "", - author = "", - comment = "", - created = "", - dockerVersion = "", - id = "", - os = "", - osVersion = "", - parent = "", - size = 0, - repoTags = null, - repoDigests = null, - virtualSize = 0 - ) - ) - return@forEach - } - - // 查询镜像详细信息 - val imageInfo = dockerCli.inspectImageCmd(imageName).exec() - logger.info("imageInfo: $imageInfo") - imageInspectList.add( - CheckDockerImageResponse( - errorCode = 0, - errorMessage = "", - arch = imageInfo.arch, - author = imageInfo.author, - comment = imageInfo.comment, - created = imageInfo.created, - dockerVersion = imageInfo.dockerVersion, - id = imageInfo.id, - os = imageInfo.os, - osVersion = imageInfo.osVersion, - parent = imageInfo.parent, - size = imageInfo.size, - repoTags = imageInfo.repoTags, - repoDigests = imageInfo.repoDigests, - virtualSize = imageInfo.virtualSize - ) - ) - logger.info("==========================") - } - - logger.info("imageInspectList: $imageInspectList") - - return imageInspectList } fun checkRemoteDockerImage( @@ -164,7 +88,7 @@ class InspectImageService @Autowired constructor( ) .build() - OkhttpUtils.doHttp(request).use { response -> + OkhttpUtils.doLongHttp(request).use { response -> val responseContent = response.body!!.string() logger.info("$userId check remoteImage: $responseContent") if (!response.isSuccessful) { @@ -182,36 +106,47 @@ class InspectImageService @Autowired constructor( } } - inner class MyPullImageResultCallback internal constructor( - private val userId: String - ) : PullImageResultCallback() { - private val totalList = mutableListOf() - private val step = mutableMapOf() - override fun onNext(item: PullResponseItem?) { - val text = item?.progressDetail - if (null != text && text.current != null && text.total != 0L) { - val lays = if (!totalList.contains(text.total!!)) { - totalList.add(text.total!!) - totalList.size + 1 - } else { - totalList.indexOf(text.total!!) + 1 - } - var currentProgress = text.current!! * 100 / text.total!! - if (currentProgress > 100) { - currentProgress = 100 - } + fun checkKubernetesDockerImage( + userId: String, + checkDockerImageRequestList: List + ): List { + try { + val dispatchCheckDockerImageRequestList = + checkDockerImageRequestList.stream().map { checkDockerImageRequest -> + com.tencent.devops.dispatch.kubernetes.pojo.CheckDockerImageRequest( + imageName = checkDockerImageRequest.imageName, + registryHost = checkDockerImageRequest.registryHost, + registryUser = checkDockerImageRequest.registryUser, + registryPwd = checkDockerImageRequest.registryPwd + ) + }.collect(Collectors.toList()) - if (currentProgress >= step[lays]?.plus(25) ?: 5) { - logger.info("$userId pulling images, $lays layer, progress: $currentProgress%") - step[lays] = currentProgress - } - } - super.onNext(item) + val response = client.getWithoutRetry(ServiceDockerImageResource::class).checkDockerImage( + userId = userId, + checkDockerImageRequestList = dispatchCheckDockerImageRequestList + ).data + + return response?.stream()?.map { + CheckDockerImageResponse( + errorCode = it.errorCode, + errorMessage = it.errorMessage, + arch = it.arch, + author = it.author, + comment = it.comment, + created = it.created, + dockerVersion = it.dockerVersion, + id = it.id, + os = it.os, + osVersion = it.osVersion, + parent = it.parent, + size = it.size, + repoTags = it.repoTags, + repoDigests = it.repoDigests, + virtualSize = it.virtualSize + ) }?.collect(Collectors.toList()) ?: emptyList() + } catch (e: Exception) { + logger.error("Check dispatch image error: ${e.message}") + return emptyList() } } } - -// fun main(args: Array) { -// println(SecurityUtil.decrypt("7Rq3q4+3wRSkYX78nrcWNw==")) -// println(SecurityUtil.encrypt("!@#098Bcs")) -// } diff --git a/src/backend/ci/core/misc/biz-plugin/src/main/kotlin/com/tencent/devops/plugin/service/ScmCheckService.kt b/src/backend/ci/core/misc/biz-plugin/src/main/kotlin/com/tencent/devops/plugin/service/ScmCheckService.kt index dce936bc78a..f5e9395f517 100644 --- a/src/backend/ci/core/misc/biz-plugin/src/main/kotlin/com/tencent/devops/plugin/service/ScmCheckService.kt +++ b/src/backend/ci/core/misc/biz-plugin/src/main/kotlin/com/tencent/devops/plugin/service/ScmCheckService.kt @@ -35,7 +35,9 @@ import com.tencent.devops.common.api.exception.OperationException import com.tencent.devops.common.api.exception.ParamBlankException import com.tencent.devops.common.api.util.DHUtil import com.tencent.devops.common.api.util.EnvUtils +import com.tencent.devops.common.api.util.timestampmilli import com.tencent.devops.common.client.Client +import com.tencent.devops.common.pipeline.enums.StartType import com.tencent.devops.plugin.api.pojo.GitCommitCheckEvent import com.tencent.devops.plugin.utils.QualityUtils import com.tencent.devops.process.utils.Credential @@ -52,6 +54,7 @@ import com.tencent.devops.repository.pojo.GithubCheckRunsResponse import com.tencent.devops.repository.pojo.GithubRepository import com.tencent.devops.repository.pojo.Repository import com.tencent.devops.repository.pojo.enums.RepoAuthType +import com.tencent.devops.repository.sdk.github.pojo.CheckRunOutput import com.tencent.devops.scm.pojo.CommitCheckRequest import com.tencent.devops.scm.pojo.RepoSessionRequest import com.tencent.devops.ticket.api.ServiceCredentialResource @@ -60,6 +63,9 @@ import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service import java.net.URLEncoder +import java.time.LocalDateTime +import java.time.ZoneId +import java.time.format.DateTimeFormatter import java.util.Base64 import javax.ws.rs.NotFoundException @@ -118,7 +124,16 @@ class ScmCheckService @Autowired constructor(private val client: Client) { description = description, block = block, mrRequestId = event.mergeRequestId, - reportData = QualityUtils.getQualityGitMrResult(client, event), + reportData = QualityUtils.getQualityGitMrResult( + client = client, + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + startTime = startTime, + eventStatus = status, + triggerType = triggerType, + scmType = repo.getScmType() + ), targetBranch = targetBranch ) if (isOauth) { @@ -132,15 +147,17 @@ class ScmCheckService @Autowired constructor(private val client: Client) { fun addGithubCheckRuns( projectId: String, + pipelineId: String, + buildId: String, repositoryConfig: RepositoryConfig, name: String, commitId: String, detailUrl: String, externalId: String, status: String, - startedAt: String?, + startedAt: LocalDateTime?, conclusion: String?, - completedAt: String? + completedAt: LocalDateTime? ): GithubCheckRunsResponse { logger.info("Project($projectId) add github commit($commitId) check runs") @@ -154,9 +171,9 @@ class ScmCheckService @Autowired constructor(private val client: Client) { detailsUrl = detailUrl, externalId = externalId, status = status, - startedAt = startedAt, + startedAt = startedAt?.atZone(ZoneId.systemDefault())?.format(DateTimeFormatter.ISO_INSTANT), conclusion = conclusion, - completedAt = completedAt + completedAt = completedAt?.atZone(ZoneId.systemDefault())?.format(DateTimeFormatter.ISO_INSTANT) ) return client.get(ServiceGithubResource::class).addCheckRuns( @@ -169,15 +186,18 @@ class ScmCheckService @Autowired constructor(private val client: Client) { fun updateGithubCheckRuns( checkRunId: Long, projectId: String, + pipelineId: String, + buildId: String, repositoryConfig: RepositoryConfig, name: String, commitId: String, detailUrl: String, externalId: String, status: String, - startedAt: String?, + startedAt: LocalDateTime?, conclusion: String?, - completedAt: String? + completedAt: LocalDateTime?, + pipelineName: String ) { logger.info("Project($projectId) update github commit($commitId) check runs") @@ -191,9 +211,24 @@ class ScmCheckService @Autowired constructor(private val client: Client) { detailsUrl = detailUrl, externalId = externalId, status = status, - startedAt = startedAt, + startedAt = startedAt?.atZone(ZoneId.systemDefault())?.format(DateTimeFormatter.ISO_INSTANT), conclusion = conclusion, - completedAt = completedAt + completedAt = completedAt?.atZone(ZoneId.systemDefault())?.format(DateTimeFormatter.ISO_INSTANT), + output = CheckRunOutput( + summary = "This check concluded as $conclusion.", + text = null, // github + title = pipelineName, + reportData = QualityUtils.getQualityGitMrResult( + client = client, + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + startTime = startedAt?.timestampmilli() ?: 0L, + eventStatus = status, + triggerType = StartType.WEB_HOOK.name, + scmType = ScmType.GITHUB + ) + ) ) client.get(ServiceGithubResource::class).updateCheckRuns( diff --git a/src/backend/ci/core/misc/biz-plugin/src/main/kotlin/com/tencent/devops/plugin/service/git/CodeWebhookService.kt b/src/backend/ci/core/misc/biz-plugin/src/main/kotlin/com/tencent/devops/plugin/service/git/CodeWebhookService.kt index 349e2434eed..620e9e4e609 100644 --- a/src/backend/ci/core/misc/biz-plugin/src/main/kotlin/com/tencent/devops/plugin/service/git/CodeWebhookService.kt +++ b/src/backend/ci/core/misc/biz-plugin/src/main/kotlin/com/tencent/devops/plugin/service/git/CodeWebhookService.kt @@ -76,7 +76,6 @@ import org.springframework.stereotype.Service import java.time.Instant import java.time.LocalDateTime import java.time.ZoneId -import java.time.format.DateTimeFormatter @Service @Suppress("ALL") @@ -159,7 +158,7 @@ class CodeWebhookService @Autowired constructor( repositoryConfig = repositoryConfig, commitId = commitId, status = status, - startedAt = null, + startedAt = (event.startTime ?: 0L) / 1000, // 毫秒 -> 秒 conclusion = conclusion, completedAt = LocalDateTime.now().timestamp(), userId = event.userId, @@ -207,8 +206,10 @@ class CodeWebhookService @Autowired constructor( return } - if (variables[PIPELINE_START_CHANNEL] != ChannelCode.BS.name) { - logger.warn("Process instance($buildId) is not bs channel") + if (variables[PIPELINE_START_CHANNEL] != ChannelCode.BS.name && + variables[PIPELINE_START_CHANNEL] != ChannelCode.GONGFENGSCAN.name + ) { + logger.warn("Process instance($buildId) is not bs or gongfengscan channel") return } // 发布瞬间，重试的构建由process服务处理 @@ -543,6 +544,8 @@ class CodeWebhookService @Autowired constructor( val checkRunId = if (conclusion == null) { val result = scmCheckService.addGithubCheckRuns( projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, repositoryConfig = repositoryConfig, name = record.checkRunName ?: "$pipelineName #$buildNum", commitId = commitId, @@ -562,6 +565,9 @@ class CodeWebhookService @Autowired constructor( scmCheckService.updateGithubCheckRuns( checkRunId = record.checkRunId, projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + pipelineName = pipelineName, repositoryConfig = repositoryConfig, // 兼容历史数据 name = record.checkRunName ?: "$pipelineName #$buildNum", diff --git a/src/backend/ci/core/misc/biz-plugin/src/main/kotlin/com/tencent/devops/plugin/utils/QualityUtils.kt b/src/backend/ci/core/misc/biz-plugin/src/main/kotlin/com/tencent/devops/plugin/utils/QualityUtils.kt index 81beaf3e557..f01848e2eb4 100644 --- a/src/backend/ci/core/misc/biz-plugin/src/main/kotlin/com/tencent/devops/plugin/utils/QualityUtils.kt +++ b/src/backend/ci/core/misc/biz-plugin/src/main/kotlin/com/tencent/devops/plugin/utils/QualityUtils.kt @@ -27,13 +27,13 @@ package com.tencent.devops.plugin.utils +import com.tencent.devops.common.api.enums.ScmType import com.tencent.devops.common.api.util.DateTimeUtil import com.tencent.devops.common.client.Client import com.tencent.devops.common.pipeline.enums.StartType import com.tencent.devops.common.quality.pojo.enums.QualityOperation import com.tencent.devops.common.service.utils.HomeHostUtil import com.tencent.devops.common.web.utils.I18nUtil -import com.tencent.devops.plugin.api.pojo.GitCommitCheckEvent import com.tencent.devops.plugin.codecc.CodeccUtils import com.tencent.devops.plugin.constant.PluginMessageCode.BK_CI_PIPELINE import com.tencent.devops.process.api.service.ServicePipelineResource @@ -45,31 +45,58 @@ import com.tencent.devops.quality.constant.codeccToolUrlPathMap @Suppress("ALL") object QualityUtils { + /** + * 获取质量红线结果 + * @param client + * @param projectId 项目ID + * @param pipelineId 流水线ID + * @param buildId 流水线构建ID + * @param eventStatus 事件状态 + * @param startTime 事件开始时间 + * @param triggerType 触发类型 + * @param insertUrl 是否插入链接地址[github 不需要插入链接] + */ fun getQualityGitMrResult( client: Client, - event: GitCommitCheckEvent + projectId: String, + pipelineId: String, + buildId: String, + eventStatus: String, + startTime: Long, + triggerType: String, + scmType: ScmType ): Pair, MutableMap>>> { - val projectId = event.projectId - val pipelineId = event.pipelineId - val buildId = event.buildId val pipelineName = client.get(ServicePipelineResource::class) - .getPipelineNameByIds(projectId, setOf(pipelineId)) - .data?.get(pipelineId) ?: "" - - val titleData = mutableListOf(event.status, - DateTimeUtil.formatMilliTime(System.currentTimeMillis() - event.startTime), - StartType.toReadableString( - event.triggerType, - null, - I18nUtil.getLanguage(I18nUtil.getRequestUserId()) - ), - pipelineName, - "${HomeHostUtil.innerServerHost()}/console/pipeline/$projectId/$pipelineId/detail/$buildId", - I18nUtil.getCodeLanMessage(BK_CI_PIPELINE) + .getPipelineNameByIds(projectId, setOf(pipelineId)) + .data?.get(pipelineId) ?: "" + // github 不需要插入链接, 仅在插件名处插入链接，链接地址用codecc插件输出变量 + val githubRepo = scmType == ScmType.GITHUB + val titleData = mutableListOf( + eventStatus, + DateTimeUtil.formatMilliTime(System.currentTimeMillis() - startTime), + StartType.toReadableString( + triggerType, + null, + I18nUtil.getLanguage(I18nUtil.getRequestUserId()) + ), + pipelineName, + if (githubRepo) { + "" + } else { + "${HomeHostUtil.innerServerHost()}/console/pipeline/$projectId/$pipelineId/detail/$buildId" + }, + I18nUtil.getCodeLanMessage(BK_CI_PIPELINE) ) val ruleName = mutableSetOf() - + // 插件输出变量 + val reportUrl = getBuildVar( + client = client, + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + varName = CodeccUtils.BK_CI_CODECC_REPORT_URL + ) // key：质量红线产出插件 // value：指标、预期、结果、状态 val resultMap = mutableMapOf>>() @@ -79,10 +106,18 @@ object QualityUtils { val indicator = client.get(ServiceQualityIndicatorResource::class) .get(projectId, interceptItem.indicatorId).data val indicatorElementName = indicator?.elementType ?: "" - val elementCnName = ElementUtils.getElementCnName(indicatorElementName, projectId) + + val elementCnName = ElementUtils.getElementCnName(indicatorElementName, projectId).let { + if (githubRepo && !reportUrl.isNullOrBlank()) { + "$it" + } else { + it + } + } val resultList = resultMap[elementCnName] ?: mutableListOf() - val actualValue = if (CodeccUtils.isCodeccAtom(indicatorElementName)) { - getActualValue( + val actualValue = when { + githubRepo -> interceptItem.actualValue ?: "null" + CodeccUtils.isCodeccAtom(indicatorElementName) -> getActualValue( projectId = projectId, pipelineId = pipelineId, buildId = buildId, @@ -90,8 +125,7 @@ object QualityUtils { value = interceptItem.actualValue ?: "null", client = client ) - } else { - interceptItem.actualValue ?: "null" + else -> interceptItem.actualValue ?: "null" } resultList.add( listOf( @@ -118,13 +152,13 @@ object QualityUtils { value: String, client: Client ): String { - val variable = client.get(ServiceVarResource::class).getBuildVar( + val taskId = getBuildVar( + client = client, projectId = projectId, pipelineId = pipelineId, buildId = buildId, varName = CodeccUtils.BK_CI_CODECC_TASK_ID - ).data - val taskId = variable?.get(CodeccUtils.BK_CI_CODECC_TASK_ID) + ) return if (detail.isNullOrBlank() || detail.split(",").size > 1) { "$value" @@ -137,4 +171,21 @@ object QualityUtils { "$value" } } + + private fun getBuildVar( + client: Client, + projectId: String, + pipelineId: String, + buildId: String, + varName: String + ): String? { + val variable = client.get(ServiceVarResource::class).getBuildVar( + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + varName = varName + ).data + val taskId = variable?.get(varName) + return taskId + } } diff --git a/src/backend/ci/core/notify/biz-notify/src/main/kotlin/com/tencent/devops/notify/service/notifier/WeworkGroupNotifier.kt b/src/backend/ci/core/notify/biz-notify/src/main/kotlin/com/tencent/devops/notify/service/notifier/WeworkGroupNotifier.kt index 2aac7ff87ee..14d86a869f2 100644 --- a/src/backend/ci/core/notify/biz-notify/src/main/kotlin/com/tencent/devops/notify/service/notifier/WeworkGroupNotifier.kt +++ b/src/backend/ci/core/notify/biz-notify/src/main/kotlin/com/tencent/devops/notify/service/notifier/WeworkGroupNotifier.kt @@ -31,7 +31,7 @@ class WeworkGroupNotifier @Autowired constructor( commonNotifyMessageTemplateRecord: TCommonNotifyMessageTemplateRecord ) { logger.info("send WEWORK_GROUP msg: $commonNotifyMessageTemplateRecord.id") - val groups = request.bodyParams?.get(NotifyUtils.WEWORK_GROUP_KEY)?.split(",") + val groups = request.bodyParams?.get(NotifyUtils.WEWORK_GROUP_KEY)?.split("[,;]".toRegex()) if (groups.isNullOrEmpty()) { logger.info("wework group is empty, so return.") return diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwAuthProjectResourceV4.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwAuthProjectResourceV4.kt index eb9f3248efa..0efbdfe7396 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwAuthProjectResourceV4.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwAuthProjectResourceV4.kt @@ -1,5 +1,7 @@ package com.tencent.devops.openapi.api.apigw.v4 +import com.tencent.devops.auth.pojo.dto.GroupAddDTO +import com.tencent.devops.auth.pojo.vo.GroupPermissionDetailVo import com.tencent.devops.auth.pojo.vo.ProjectPermissionInfoVO import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_APP_CODE import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_APP_CODE_DEFAULT_VALUE @@ -7,6 +9,7 @@ import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_USER_ID import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.auth.api.AuthResourceType import com.tencent.devops.common.auth.api.pojo.BkAuthGroup +import com.tencent.devops.common.auth.api.pojo.BkAuthGroupAndUserList import com.tencent.devops.project.pojo.ProjectCreateUserInfo import com.tencent.devops.project.pojo.ProjectDeleteUserInfo import io.swagger.v3.oas.annotations.Operation @@ -86,6 +89,45 @@ interface ApigwAuthProjectResourceV4 { group: BkAuthGroup? = null ): Result> + @GET + @Path("/get_project_group_and_users") + @Operation(summary = "获取项目组成员", tags = ["v4_app_get_project_group_and_users"]) + fun getProjectGroupAndUserList( + @Parameter(description = "appCode", required = true, example = AUTH_HEADER_DEVOPS_APP_CODE_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_DEVOPS_APP_CODE) + appCode: String?, + @Parameter(description = "apigw Type", required = true) + @PathParam("apigwType") + apigwType: String?, + @Parameter(description = "userId") + @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) + userId: String?, + @Parameter(description = "projectId", required = true) + @PathParam("projectId") + projectId: String + ): Result> + + @GET + @Path("/{groupId}/get_group_permission_detail") + @Operation(summary = "查询用户组权限详情", tags = ["v4_app_get_group_permission_detail"]) + fun getGroupPermissionDetail( + @Parameter(description = "appCode", required = true, example = AUTH_HEADER_DEVOPS_APP_CODE_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_DEVOPS_APP_CODE) + appCode: String?, + @Parameter(description = "apigw Type", required = true) + @PathParam("apigwType") + apigwType: String?, + @Parameter(description = "userId") + @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) + userId: String?, + @Parameter(description = "projectId", required = true) + @PathParam("projectId") + projectId: String, + @Parameter(description = "用户组ID") + @PathParam("groupId") + groupId: Int + ): Result>> + @POST @Path("/batch_add_resource_group_members") @Operation(summary = "用户组批量添加成员", tags = ["v4_app_batch_add_resource_group_members"]) @@ -138,7 +180,7 @@ interface ApigwAuthProjectResourceV4 { apigwType: String?, @Parameter(description = "userId") @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) - userId: String, + userId: String?, @Parameter(description = "项目Id", required = true) @PathParam("projectId") projectId: String, @@ -150,6 +192,26 @@ interface ApigwAuthProjectResourceV4 { groupCode: BkAuthGroup ): Result + @POST + @Path("/create_group") + @Operation(summary = "创建自定义组(不包含权限，空权限组)", tags = ["v4_app_create_group"]) + fun createGroup( + @Parameter(description = "appCode", required = true, example = AUTH_HEADER_DEVOPS_APP_CODE_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_DEVOPS_APP_CODE) + appCode: String?, + @Parameter(description = "apigw Type", required = true) + @PathParam("apigwType") + apigwType: String?, + @Parameter(description = "userId") + @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) + userId: String?, + @Parameter(description = "项目Id", required = true) + @PathParam("projectId") + projectId: String, + @Parameter(description = "添加用户组实体", required = true) + groupAddDTO: GroupAddDTO + ): Result + @DELETE @Path("/delete_group/{resourceType}") @Operation(summary = "刪除用户组", tags = ["v4_app_delete_group"]) @@ -162,7 +224,7 @@ interface ApigwAuthProjectResourceV4 { apigwType: String?, @Parameter(description = "userId") @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) - userId: String, + userId: String?, @Parameter(description = "项目Id", required = true) @PathParam("projectId") projectId: String, diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwAuthValidateResourceV4.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwAuthValidateResourceV4.kt index 60e23c34c65..ea76acc1879 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwAuthValidateResourceV4.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwAuthValidateResourceV4.kt @@ -5,9 +5,9 @@ import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_APP_CODE_DEFAULT_VA import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_USER_ID import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.auth.api.pojo.BkAuthGroup -import io.swagger.v3.oas.annotations.tags.Tag import io.swagger.v3.oas.annotations.Operation import io.swagger.v3.oas.annotations.Parameter +import io.swagger.v3.oas.annotations.tags.Tag import javax.ws.rs.Consumes import javax.ws.rs.GET import javax.ws.rs.HeaderParam @@ -46,4 +46,25 @@ interface ApigwAuthValidateResourceV4 { @Parameter(description = "用户组类型", required = false) group: BkAuthGroup? = null ): Result + + @GET + @Path("/check_user_in_project_level_group/{userId}") + @Operation( + summary = "检查用户是否在项目级别的用户组中", + tags = ["v4_app_check_user_in_project_level_group"] + ) + fun checkUserInProjectLevelGroup( + @Parameter(description = "appCode", required = true, example = AUTH_HEADER_DEVOPS_APP_CODE_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_DEVOPS_APP_CODE) + appCode: String?, + @Parameter(description = "apigw Type", required = true) + @PathParam("apigwType") + apigwType: String?, + @PathParam("projectId") + @Parameter(description = "项目ID", required = true) + projectId: String, + @PathParam("userId") + @Parameter(description = "用户Id", required = true) + userId: String + ): Result } diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwBuildResourceV4.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwBuildResourceV4.kt index da13e54e453..59d88faf91b 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwBuildResourceV4.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwBuildResourceV4.kt @@ -234,19 +234,19 @@ interface ApigwBuildResourceV4 { ) @QueryParam("updateTimeDesc") updateTimeDesc: Boolean? = null, - @Parameter(description = "代码库别名", required = false) + @Parameter(description = "源材料代码库别名", required = false) @QueryParam("materialAlias") materialAlias: List?, @Parameter(description = "代码库URL", required = false) @QueryParam("materialUrl") materialUrl: String?, - @Parameter(description = "分支", required = false) + @Parameter(description = "源材料分支", required = false) @QueryParam("materialBranch") materialBranch: List?, - @Parameter(description = "commitId", required = false) + @Parameter(description = "源材料commitId", required = false) @QueryParam("materialCommitId") materialCommitId: String?, - @Parameter(description = "commitMessage", required = false) + @Parameter(description = "源材料commitMessage", required = false) @QueryParam("materialCommitMessage") materialCommitMessage: String?, @Parameter(description = "状态", required = false) @@ -296,7 +296,13 @@ interface ApigwBuildResourceV4 { startUser: List?, @Parameter(description = "是否查询归档数据", required = false) @QueryParam("archiveFlag") - archiveFlag: Boolean? = false + archiveFlag: Boolean? = false, + @Parameter(description = "触发代码库", required = false) + @QueryParam("triggerAlias") + triggerAlias: List? = null, + @Parameter(description = "触发分支", required = false) + @QueryParam("triggerBranch") + triggerBranch: List? = null ): Result> @Operation(summary = "获取流水线手动启动参数", tags = ["v4_app_build_startInfo", "v4_user_build_startInfo"]) diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwRepositoryResourceV4.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwRepositoryResourceV4.kt index 6653a81555f..8d9ae930658 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwRepositoryResourceV4.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/ApigwRepositoryResourceV4.kt @@ -243,4 +243,46 @@ interface ApigwRepositoryResourceV4 { @QueryParam("repositoryType") repositoryType: RepositoryType? ): Result + + @Operation(summary = "开启PAC", tags = ["v4_app_repository_enable_pac", "v4_user_repository_enable_pac"]) + @PUT + @Path("/{repositoryHashId}/pac/enable") + fun enablePac( + @Parameter(description = "appCode", required = true, example = AUTH_HEADER_DEVOPS_APP_CODE_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_DEVOPS_APP_CODE) + appCode: String?, + @Parameter(description = "apigw Type", required = true) + @PathParam("apigwType") + apigwType: String?, + @Parameter(description = "用户ID", required = true, example = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) + userId: String, + @Parameter(description = "项目ID(项目英文名)", required = true) + @PathParam("projectId") + projectId: String, + @Parameter(description = "代码库哈希ID", required = true) + @PathParam("repositoryHashId") + repositoryHashId: String + ): Result + + @Operation(summary = "关闭PAC", tags = ["v4_app_repository_disable_pac", "v4_user_repository_disable_pac"]) + @PUT + @Path("/{repositoryHashId}/pac/disable") + fun disablePac( + @Parameter(description = "appCode", required = true, example = AUTH_HEADER_DEVOPS_APP_CODE_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_DEVOPS_APP_CODE) + appCode: String?, + @Parameter(description = "apigw Type", required = true) + @PathParam("apigwType") + apigwType: String?, + @Parameter(description = "用户ID", required = true, example = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) + userId: String, + @Parameter(description = "项目ID(项目英文名)", required = true) + @PathParam("projectId") + projectId: String, + @Parameter(description = "代码库哈希ID", required = true) + @PathParam("repositoryHashId") + repositoryHashId: String + ): Result } diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/environment/ApigwEnvironmentAgentResourceV4.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/environment/ApigwEnvironmentAgentResourceV4.kt index b22cde96578..2e50ac6943f 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/environment/ApigwEnvironmentAgentResourceV4.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/api/apigw/v4/environment/ApigwEnvironmentAgentResourceV4.kt @@ -33,9 +33,12 @@ import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_USER_ID import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE import com.tencent.devops.common.api.pojo.Page import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.environment.pojo.EnvVar import com.tencent.devops.environment.pojo.NodeBaseInfo import com.tencent.devops.environment.pojo.NodeWithPermission import com.tencent.devops.environment.pojo.thirdpartyagent.AgentBuildDetail +import com.tencent.devops.environment.pojo.thirdpartyagent.BatchFetchAgentData +import com.tencent.devops.environment.pojo.thirdpartyagent.BatchUpdateAgentEnvVar import com.tencent.devops.environment.pojo.thirdpartyagent.ThirdPartyAgentDetail import io.swagger.v3.oas.annotations.tags.Tag import io.swagger.v3.oas.annotations.Operation @@ -43,6 +46,7 @@ import io.swagger.v3.oas.annotations.Parameter import javax.ws.rs.Consumes import javax.ws.rs.GET import javax.ws.rs.HeaderParam +import javax.ws.rs.POST import javax.ws.rs.Path import javax.ws.rs.PathParam import javax.ws.rs.Produces @@ -166,4 +170,50 @@ interface ApigwEnvironmentAgentResourceV4 { @QueryParam("pageSize") pageSize: Int? ): Result> + + @Operation( + summary = "批量查询Agent环境变量", + tags = ["v4_user_node_third_part_agent_envvar", "v4_app_node_third_part_agent_envvar"] + ) + @POST + @Path("/fetch_agent_env") + fun fetchAgentEnv( + @Parameter(description = "appCode", required = true, example = AUTH_HEADER_DEVOPS_APP_CODE_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_DEVOPS_APP_CODE) + appCode: String?, + @Parameter(description = "apigw Type", required = true) + @PathParam("apigwType") + apigwType: String?, + @Parameter(description = "用户ID", required = true, example = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) + userId: String, + @Parameter(description = "项目ID", required = true) + @PathParam("projectId") + projectId: String, + @Parameter(description = "查询数据", required = true) + data: BatchFetchAgentData + ): Result>> + + @Operation( + summary = "批量修改Agent环境变量", + tags = ["v4_user_node_third_part_agent_update_envvar", "v4_app_node_third_part_agent_update_envvar"] + ) + @POST + @Path("/batch_update_agent_env") + fun batchUpdateEnv( + @Parameter(description = "appCode", required = true, example = AUTH_HEADER_DEVOPS_APP_CODE_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_DEVOPS_APP_CODE) + appCode: String?, + @Parameter(description = "apigw Type", required = true) + @PathParam("apigwType") + apigwType: String?, + @Parameter(description = "用户ID", required = true, example = AUTH_HEADER_DEVOPS_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_DEVOPS_USER_ID) + userId: String, + @Parameter(description = "项目ID", required = true) + @PathParam("projectId") + projectId: String, + @Parameter(description = "修改数据", required = true) + data: BatchUpdateAgentEnvVar + ): Result } diff --git a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/constant/OpenAPIMessageCode.kt b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/constant/OpenAPIMessageCode.kt index ccc3dfdad92..de4d3f7f217 100644 --- a/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/constant/OpenAPIMessageCode.kt +++ b/src/backend/ci/core/openapi/api-openapi/src/main/kotlin/com/tencent/devops/openapi/constant/OpenAPIMessageCode.kt @@ -91,4 +91,5 @@ object OpenAPIMessageCode { const val BK_THE_FIELD_IS_READ_ONLY = "bkTheFieldIsReadOnly" // 该字段只读 const val BK_OBJECT_PROPERTY_ILLUSTRATE = "bkObjectPropertyIllustrate" // Any 任意类型，参照实际请求或返回 const val BK_NO_SUCH_PARAMETER = "bkNoSuchParameter" // 无此参数 + const val APP_CODE_PERMISSION_DENIED_MESSAGE = "appCodePermissionDeniedMessage" } diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt index 3bf300210db..852b472560f 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/aspect/ApiAspect.kt @@ -37,6 +37,7 @@ import com.tencent.devops.common.redis.RedisOperation import com.tencent.devops.common.service.BkTag import com.tencent.devops.common.web.utils.I18nUtil import com.tencent.devops.openapi.IgnoreProjectId +import com.tencent.devops.openapi.constant.OpenAPIMessageCode import com.tencent.devops.openapi.constant.OpenAPIMessageCode.PARAM_VERIFY_FAIL import com.tencent.devops.openapi.es.ESMessage import com.tencent.devops.openapi.es.IESService @@ -183,7 +184,11 @@ class ApiAspect( if (appCode != null && apigwType == "apigw-app" && !appCodeService.validAppCode(appCode, projectId)) { throw PermissionForbiddenException( - message = "Permission denied: apigwType[$apigwType],appCode[$appCode],ProjectId[$projectId]" + message = "Permission denied: apigwType[$apigwType]," + + "appCode[$appCode],ProjectId[$projectId] " + I18nUtil.getCodeLanMessage( + messageCode = OpenAPIMessageCode.APP_CODE_PERMISSION_DENIED_MESSAGE, + language = I18nUtil.getLanguage(userId) + ) ) } } diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v3/ApigwPipelineResourceV3Impl.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v3/ApigwPipelineResourceV3Impl.kt index 3ca4f15afe9..bfe01fd8073 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v3/ApigwPipelineResourceV3Impl.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v3/ApigwPipelineResourceV3Impl.kt @@ -180,7 +180,8 @@ class ApigwPipelineResourceV3Impl @Autowired constructor( userId = userId, projectId = projectId, pipelineId = pipelineId, - channelCode = apiGatewayUtil.getChannelCode() + channelCode = apiGatewayUtil.getChannelCode(), + checkFlag = true ) } diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwAuthProjectResourceV4Impl.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwAuthProjectResourceV4Impl.kt index 9b8a0923d7d..0e187c02f3a 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwAuthProjectResourceV4Impl.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwAuthProjectResourceV4Impl.kt @@ -3,10 +3,13 @@ package com.tencent.devops.openapi.resources.apigw.v4 import com.tencent.devops.auth.api.service.ServiceProjectAuthResource import com.tencent.devops.auth.api.service.ServiceResourceGroupResource import com.tencent.devops.auth.api.service.ServiceResourceMemberResource +import com.tencent.devops.auth.pojo.dto.GroupAddDTO +import com.tencent.devops.auth.pojo.vo.GroupPermissionDetailVo import com.tencent.devops.auth.pojo.vo.ProjectPermissionInfoVO import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.auth.api.AuthResourceType import com.tencent.devops.common.auth.api.pojo.BkAuthGroup +import com.tencent.devops.common.auth.api.pojo.BkAuthGroupAndUserList import com.tencent.devops.common.client.Client import com.tencent.devops.common.client.ClientTokenService import com.tencent.devops.common.web.RestResource @@ -30,7 +33,7 @@ class ApigwAuthProjectResourceV4Impl @Autowired constructor( apigwType: String?, projectId: String ): Result { - logger.info("OPENAPI_AUTH_PROJECT_PERMISSION_INFO_V4 getProjectPermissionInfo|$projectId") + logger.info("OPENAPI_AUTH_PROJECT_RESOURCE_V4 getProjectPermissionInfo|$appCode|$projectId") return client.get(ServiceProjectAuthResource::class).getProjectPermissionInfo( token = tokenService.getSystemToken(), projectCode = projectId @@ -45,7 +48,7 @@ class ApigwAuthProjectResourceV4Impl @Autowired constructor( resourceCode: String, group: BkAuthGroup? ): Result> { - logger.info("OPENAPI_AUTH_PROJECT_PERMISSION_INFO_V4 getResourceGroupUsers|$projectId") + logger.info("OPENAPI_AUTH_PROJECT_RESOURCE_V4 getResourceGroupUsers|$appCode|$projectId") return client.get(ServiceResourceMemberResource::class).getResourceGroupMembers( token = tokenService.getSystemToken(), projectCode = projectId, @@ -55,6 +58,33 @@ class ApigwAuthProjectResourceV4Impl @Autowired constructor( ) } + override fun getProjectGroupAndUserList( + appCode: String?, + apigwType: String?, + userId: String?, + projectId: String + ): Result> { + logger.info("OPENAPI_AUTH_PROJECT_RESOURCE_V4 getResourceGroupUsers|$appCode|$userId|$projectId") + return client.get(ServiceProjectAuthResource::class).getProjectGroupAndUserList( + token = tokenService.getSystemToken(), + projectCode = projectId + ) + } + + override fun getGroupPermissionDetail( + appCode: String?, + apigwType: String?, + userId: String?, + projectId: String, + groupId: Int + ): Result>> { + logger.info("OPENAPI_AUTH_PROJECT_RESOURCE_V4 getGroupPermissionDetail|$appCode|$userId|$projectId") + return client.get(ServiceResourceGroupResource::class).getGroupPermissionDetail( + projectCode = projectId, + groupId = groupId + ) + } + override fun batchAddResourceGroupMembers( appCode: String?, apigwType: String?, @@ -62,10 +92,12 @@ class ApigwAuthProjectResourceV4Impl @Autowired constructor( projectId: String, createInfo: ProjectCreateUserInfo ): Result { - logger.info("createProjectUser v4 |$appCode|$userId|$projectId|$createInfo") + logger.info( + "OPENAPI_AUTH_PROJECT_RESOURCE_V4 batchAddResourceGroupMembers " + + " |$appCode|$userId|$projectId|$createInfo" + ) return client.get(ServiceResourceMemberResource::class).batchAddResourceGroupMembers( token = tokenService.getSystemToken(), - userId = createInfo.createUserId, projectCode = projectId, projectCreateUserInfo = createInfo ) @@ -78,10 +110,12 @@ class ApigwAuthProjectResourceV4Impl @Autowired constructor( projectId: String, deleteInfo: ProjectDeleteUserInfo ): Result { - logger.info("deleteProjectUser v4 |$appCode|$userId|$projectId|$deleteInfo") + logger.info( + "OPENAPI_AUTH_PROJECT_RESOURCE_V4 batchDeleteResourceGroupMembers" + + "|$appCode|$userId|$projectId|$deleteInfo" + ) return client.get(ServiceResourceMemberResource::class).batchDeleteResourceGroupMembers( token = tokenService.getSystemToken(), - userId = deleteInfo.operator, projectCode = projectId, projectDeleteUserInfo = deleteInfo ) @@ -90,31 +124,46 @@ class ApigwAuthProjectResourceV4Impl @Autowired constructor( override fun createGroupByGroupCode( appCode: String?, apigwType: String?, - userId: String, + userId: String?, projectId: String, resourceType: String, groupCode: BkAuthGroup ): Result { - logger.info("createGroupByGroupCode v4 |$appCode|$userId|$projectId|$resourceType|$groupCode") + logger.info( + "OPENAPI_AUTH_PROJECT_RESOURCE_V4 createGroupByGroupCode " + + " |$appCode|$userId|$projectId|$resourceType|$groupCode" + ) return client.get(ServiceResourceGroupResource::class).createGroupByGroupCode( - userId = userId, projectCode = projectId, resourceType = resourceType, groupCode = groupCode ) } + override fun createGroup( + appCode: String?, + apigwType: String?, + userId: String?, + projectId: String, + groupAddDTO: GroupAddDTO + ): Result { + logger.info("OPENAPI_AUTH_PROJECT_RESOURCE_V4 createGroup|$appCode|$userId|$projectId|$groupAddDTO") + return client.get(ServiceResourceGroupResource::class).createGroup( + projectCode = projectId, + groupAddDTO = groupAddDTO + ) + } + override fun deleteGroup( appCode: String?, apigwType: String?, - userId: String, + userId: String?, projectId: String, resourceType: String, groupId: Int ): Result { - logger.info("deleteGroup v4 |$appCode|$userId|$projectId|$resourceType|$groupId") + logger.info("OPENAPI_AUTH_PROJECT_RESOURCE_V4 deleteGroup|$appCode|$userId|$projectId|$resourceType|$groupId") return client.get(ServiceResourceGroupResource::class).deleteGroup( - userId = userId, projectCode = projectId, resourceType = resourceType, groupId = groupId diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwAuthValidateResourceV4Impl.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwAuthValidateResourceV4Impl.kt index 9d32a025ce4..479a54f8bb3 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwAuthValidateResourceV4Impl.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwAuthValidateResourceV4Impl.kt @@ -25,13 +25,27 @@ class ApigwAuthValidateResourceV4Impl @Autowired constructor( ): Result { logger.info("OPENAPI_AUTH_VALIDATE_V4|$userId|is project user|$projectId|$group") return client.get(ServiceProjectAuthResource::class).isProjectUser( - token = tokenService.getSystemToken()!!, + token = tokenService.getSystemToken(), userId = userId, projectCode = projectId, group = group ) } + override fun checkUserInProjectLevelGroup( + appCode: String?, + apigwType: String?, + userId: String, + projectId: String + ): Result { + logger.info("OPENAPI_AUTH_VALIDATE_V4|$userId|check_user_in_project_level_group|$projectId") + return client.get(ServiceProjectAuthResource::class).checkUserInProjectLevelGroup( + token = tokenService.getSystemToken(), + userId = userId, + projectCode = projectId + ) + } + companion object { val logger = LoggerFactory.getLogger(ApigwAuthValidateResourceV4Impl::class.java) } diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwBuildResourceV4Impl.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwBuildResourceV4Impl.kt index 074d54d28b3..8d484cc457a 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwBuildResourceV4Impl.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwBuildResourceV4Impl.kt @@ -128,7 +128,9 @@ class ApigwBuildResourceV4Impl @Autowired constructor( buildNoEnd: Int?, buildMsg: String?, startUser: List?, - archiveFlag: Boolean? + archiveFlag: Boolean?, + triggerAlias: List?, + triggerBranch: List? ): Result> { logger.info( "OPENAPI_BUILD_V4|$userId|get history build|$projectId|$pipelineId|$page|$pageSize" + @@ -165,7 +167,9 @@ class ApigwBuildResourceV4Impl @Autowired constructor( buildNoEnd = buildNoEnd, buildMsg = buildMsg, startUser = startUser, - archiveFlag = archiveFlag + archiveFlag = archiveFlag, + triggerAlias = triggerAlias, + triggerBranch = triggerBranch ) } diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwLogResourceV4Impl.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwLogResourceV4Impl.kt index ea81dc07606..f8373c941f3 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwLogResourceV4Impl.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwLogResourceV4Impl.kt @@ -42,11 +42,11 @@ import com.tencent.devops.log.api.ServiceLogResource import com.tencent.devops.openapi.api.apigw.v4.ApigwLogResourceV4 import com.tencent.devops.openapi.service.IndexService import com.tencent.devops.process.api.service.ServiceBuildResource +import javax.ws.rs.core.MediaType +import javax.ws.rs.core.Response import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.beans.factory.annotation.Value -import javax.ws.rs.core.MediaType -import javax.ws.rs.core.Response @RestResource class ApigwLogResourceV4Impl @Autowired constructor( @@ -86,7 +86,7 @@ class ApigwLogResourceV4Impl @Autowired constructor( containerHashId = containerHashId, executeCount = executeCount, debug = debug, - jobId = jobId, + jobId = if (elementId.isNullOrBlank() && stepId.isNullOrBlank()) jobId else null, stepId = stepId, archiveFlag = archiveFlag ) @@ -128,7 +128,7 @@ class ApigwLogResourceV4Impl @Autowired constructor( tag = tag, containerHashId = containerHashId, executeCount = executeCount, - jobId = jobId, + jobId = if (tag.isNullOrBlank() && stepId.isNullOrBlank()) jobId else null, stepId = stepId, archiveFlag = archiveFlag ) @@ -164,7 +164,7 @@ class ApigwLogResourceV4Impl @Autowired constructor( tag = tag, containerHashId = containerHashId, executeCount = executeCount, - jobId = jobId, + jobId = if (tag.isNullOrBlank() && stepId.isNullOrBlank()) jobId else null, stepId = stepId, archiveFlag = archiveFlag ) @@ -192,7 +192,7 @@ class ApigwLogResourceV4Impl @Autowired constructor( if (!tag.isNullOrBlank()) path.append("&tag=$tag") if (!containerHashId.isNullOrBlank()) path.append("&containerHashId=$containerHashId") - if (!jobId.isNullOrBlank()) path.append("&jobId=$jobId") + if (!jobId.isNullOrBlank() && tag.isNullOrBlank() && stepId.isNullOrBlank()) path.append("&jobId=$jobId") if (!stepId.isNullOrBlank()) path.append("&stepId=$stepId") if (archiveFlag != null) path.append("&archiveFlag=$archiveFlag") val headers = mutableMapOf(AUTH_HEADER_USER_ID to userId, AUTH_HEADER_PROJECT_ID to projectId) diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwPipelineResourceV4Impl.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwPipelineResourceV4Impl.kt index fac79563798..43fc7b63210 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwPipelineResourceV4Impl.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwPipelineResourceV4Impl.kt @@ -200,7 +200,8 @@ class ApigwPipelineResourceV4Impl @Autowired constructor( userId = userId, projectId = projectId, pipelineId = pipelineId, - channelCode = apiGatewayUtil.getChannelCode() + channelCode = apiGatewayUtil.getChannelCode(), + checkFlag = true ) } diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwRepositoryResourceV4Impl.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwRepositoryResourceV4Impl.kt index df6157e8e97..cb645581dca 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwRepositoryResourceV4Impl.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/ApigwRepositoryResourceV4Impl.kt @@ -33,6 +33,7 @@ import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.client.Client import com.tencent.devops.common.web.RestResource import com.tencent.devops.openapi.api.apigw.v4.ApigwRepositoryResourceV4 +import com.tencent.devops.repository.api.ServiceRepositoryPacResource import com.tencent.devops.repository.api.ServiceRepositoryResource import com.tencent.devops.repository.pojo.Repository import com.tencent.devops.repository.pojo.RepositoryId @@ -126,6 +127,36 @@ class ApigwRepositoryResourceV4Impl @Autowired constructor(private val client: C ) } + override fun enablePac( + appCode: String?, + apigwType: String?, + userId: String, + projectId: String, + repositoryHashId: String + ): Result { + logger.info("OPENAPI_REPOSITORY_V4|$userId|enable PAC of repo in project|$projectId|$repositoryHashId") + return client.get(ServiceRepositoryPacResource::class).enablePac( + userId = userId, + projectId = projectId, + repositoryHashId = repositoryHashId + ) + } + + override fun disablePac( + appCode: String?, + apigwType: String?, + userId: String, + projectId: String, + repositoryHashId: String + ): Result { + logger.info("OPENAPI_REPOSITORY_V4|$userId|disable PAC of repo in project|$projectId|$repositoryHashId") + return client.get(ServiceRepositoryPacResource::class).disablePac( + userId = userId, + projectId = projectId, + repositoryHashId = repositoryHashId + ) + } + companion object { private val logger = LoggerFactory.getLogger(ApigwRepositoryResourceV4Impl::class.java) } diff --git a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/environment/ApigwEnvironmentAgentResourceV4Impl.kt b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/environment/ApigwEnvironmentAgentResourceV4Impl.kt index 4d1e70340af..15635b2f4f2 100644 --- a/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/environment/ApigwEnvironmentAgentResourceV4Impl.kt +++ b/src/backend/ci/core/openapi/biz-openapi/src/main/kotlin/com/tencent/devops/openapi/resources/apigw/v4/environment/ApigwEnvironmentAgentResourceV4Impl.kt @@ -34,10 +34,13 @@ import com.tencent.devops.common.client.Client import com.tencent.devops.common.web.RestResource import com.tencent.devops.environment.api.ServiceNodeResource import com.tencent.devops.environment.api.thirdpartyagent.ServiceThirdPartyAgentResource +import com.tencent.devops.environment.pojo.EnvVar import com.tencent.devops.environment.pojo.NodeBaseInfo import com.tencent.devops.environment.pojo.NodeWithPermission import com.tencent.devops.environment.pojo.enums.NodeType import com.tencent.devops.environment.pojo.thirdpartyagent.AgentBuildDetail +import com.tencent.devops.environment.pojo.thirdpartyagent.BatchFetchAgentData +import com.tencent.devops.environment.pojo.thirdpartyagent.BatchUpdateAgentEnvVar import com.tencent.devops.environment.pojo.thirdpartyagent.ThirdPartyAgentDetail import com.tencent.devops.openapi.api.apigw.v4.environment.ApigwEnvironmentAgentResourceV4 import com.tencent.devops.openapi.constant.OpenAPIMessageCode @@ -134,6 +137,42 @@ class ApigwEnvironmentAgentResourceV4Impl @Autowired constructor( ) } + override fun fetchAgentEnv( + appCode: String?, + apigwType: String?, + userId: String, + projectId: String, + data: BatchFetchAgentData + ): Result>> { + if ((data.agentHashIds?.size ?: 0) > 100 || (data.nodeHashIds?.size ?: 0) > 100) { + return Result(status = 1, message = "once max search node size 100", emptyMap()) + } + logger.info("OPENAPI_ENVIRONMENT_AGENT_V4|$userId|fetchAgentEnv|$projectId|$data") + return client.get(ServiceThirdPartyAgentResource::class).fetchAgentEnv( + userId = userId, + projectId = projectId, + data = data + ) + } + + override fun batchUpdateEnv( + appCode: String?, + apigwType: String?, + userId: String, + projectId: String, + data: BatchUpdateAgentEnvVar + ): Result { + if ((data.agentHashIds?.size ?: 0) > 100 || (data.nodeHashIds?.size ?: 0) > 100) { + return Result(status = 1, message = "once max update node size 100", false) + } + logger.info("OPENAPI_ENVIRONMENT_AGENT_V4|$userId|batchUpdateEnv|$projectId|$data") + return client.get(ServiceThirdPartyAgentResource::class).batchUpdateEnv( + userId = userId, + projectId = projectId, + data = data + ) + } + companion object { val logger = LoggerFactory.getLogger(ApigwEnvironmentAgentResourceV4Impl::class.java) } diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/app/AppPipelineBuildResource.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/app/AppPipelineBuildResource.kt index 71ad365fe62..34c467360c6 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/app/AppPipelineBuildResource.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/app/AppPipelineBuildResource.kt @@ -347,6 +347,12 @@ interface AppPipelineBuildResource { buildMsg: String?, @Parameter(description = "查看指定版本调试数据", required = false, example = "false") @QueryParam("version") - debugVersion: Int? = null + customVersion: Int? = null, + @Parameter(description = "触发代码库", required = false) + @QueryParam("triggerAlias") + triggerAlias: List?, + @Parameter(description = "触发分支", required = false) + @QueryParam("triggerBranch") + triggerBranch: List? ): Result> } diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/builds/BuildBuildResource.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/builds/BuildBuildResource.kt index 25e3320865f..ec5754a117a 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/builds/BuildBuildResource.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/builds/BuildBuildResource.kt @@ -65,6 +65,9 @@ interface BuildBuildResource { @Parameter(description = "流水线buildNum", required = true) @PathParam("buildNum") buildNum: String, + @Parameter(description = "查询方的当前构建ID，用户判断是否为调试记录", required = false) + @QueryParam("buildId") + buildId: String?, @Parameter(description = "渠道号，默认为BS", required = false) @QueryParam("channelCode") channelCode: ChannelCode? @@ -80,6 +83,9 @@ interface BuildBuildResource { @Parameter(description = "流水线ID", required = true) @PathParam("pipelineId") pipelineId: String, + @Parameter(description = "查询方的当前构建ID，用户判断是否为调试记录", required = false) + @QueryParam("buildId") + buildId: String?, @Parameter(description = "渠道号，默认为BS", required = false) @QueryParam("channelCode") channelCode: ChannelCode? diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/builds/BuildSubPipelineResource.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/builds/BuildSubPipelineResource.kt index 0966f3fa212..dc354308da1 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/builds/BuildSubPipelineResource.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/builds/BuildSubPipelineResource.kt @@ -28,6 +28,7 @@ package com.tencent.devops.process.api.builds import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_BUILD_ID +import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_EXECUTE_COUNT import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_PIPELINE_ID import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_PROJECT_ID import com.tencent.devops.common.api.pojo.Result @@ -100,7 +101,10 @@ interface BuildSubPipelineResource { @QueryParam("channelCode") channelCode: ChannelCode?, @Parameter(description = "启动参数", required = true) - values: Map + values: Map, + @Parameter(description = "当前流水线执行次数", required = false) + @HeaderParam(AUTH_HEADER_DEVOPS_EXECUTE_COUNT) + executeCount: Int? = 1 ): Result @Operation(summary = "从构建机启动指定项目的子流水线") @@ -133,7 +137,10 @@ interface BuildSubPipelineResource { @QueryParam("runMode") runMode: String, @Parameter(description = "启动参数", required = true) - values: Map + values: Map, + @Parameter(description = "当前流水线执行次数", required = false) + @HeaderParam(AUTH_HEADER_DEVOPS_EXECUTE_COUNT) + executeCount: Int? = 1 ): Result @Operation(summary = "获取子流水线启动参数") @@ -148,7 +155,13 @@ interface BuildSubPipelineResource { projectId: String, @Parameter(description = "流水线ID", required = false, example = "") @PathParam("pipelineId") - pipelineId: String + pipelineId: String, + @Parameter(description = "是否包含常量", required = false, example = "") + @QueryParam("includeConst") + includeConst: Boolean? = true, + @Parameter(description = "是否包含非入参", required = false, example = "") + @QueryParam("includeNotRequired") + includeNotRequired: Boolean? = true ): Result> @Operation(summary = "根据流水线名称获取流水线ID") diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/external/ExternalScmResource.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/external/ExternalScmResource.kt index 7d94a76612c..5ddbb049a65 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/external/ExternalScmResource.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/external/ExternalScmResource.kt @@ -59,6 +59,9 @@ interface ExternalScmResource { @Parameter(description = "X-Token") @HeaderParam("X-Token") secret: String? = null, + @Parameter(description = "来源类型,是否是测试请求") + @HeaderParam("X-Source-Type") + sourceType: String? = null, @Parameter(description = "X-TRACE-ID") @HeaderParam("X-TRACE-ID") traceId: String, diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/op/OpPipelineSettingResource.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/op/OpPipelineSettingResource.kt index b371b20aa99..0c455556702 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/op/OpPipelineSettingResource.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/op/OpPipelineSettingResource.kt @@ -112,4 +112,19 @@ interface OpPipelineSettingResource { @Parameter(description = "YAML流水线设置", required = true) pipelineAsCodeSettings: PipelineAsCodeSettings ): Result + + @Operation(summary = "更新构建指标配置") + @POST + @Path("/updateBuildMetricsSettings") + fun updateBuildMetricsSettings( + @Parameter(description = "用户ID", required = true, example = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @Parameter(description = "项目ID", required = true) + @QueryParam("projectId") + projectId: String, + @Parameter(description = "是否开启", required = true) + @QueryParam("enabled") + enabled: Boolean + ): Result } diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/open/OpenPipelineTaskResource.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/open/OpenPipelineTaskResource.kt new file mode 100644 index 00000000000..e65c7424465 --- /dev/null +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/open/OpenPipelineTaskResource.kt @@ -0,0 +1,65 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.process.api.open + +import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.process.pojo.open.BuildStatusInfo +import io.swagger.v3.oas.annotations.Operation +import io.swagger.v3.oas.annotations.Parameter +import io.swagger.v3.oas.annotations.tags.Tag +import javax.ws.rs.Consumes +import javax.ws.rs.GET +import javax.ws.rs.Path +import javax.ws.rs.Produces +import javax.ws.rs.QueryParam +import javax.ws.rs.core.MediaType + +@Tag(name = "OPEN_SERVICE_PIPELINE", description = "接口服务-流水线-任务资源") +@Path("/open/service/pipeline/") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +interface OpenPipelineTaskResource { + + @Operation(summary = "获取流水线指定任务的构建状态, 如果传了TaskId则查询相应任务的状态，否则就查询整体构建的状态") + @GET + @Path("/get_build_status") + fun getBuildStatus( + @Parameter(description = "项目ID", required = true) + @QueryParam("projectId") + projectId: String, + @Parameter(description = "流水线ID", required = true) + @QueryParam("pipelineId") + pipelineId: String, + @Parameter(description = "构建ID", required = true) + @QueryParam("buildId") + buildId: String, + @Parameter(description = "任务ID，可选，如果传了则查询相应任务的状态，否则就查询整体构建的状态", required = false) + @QueryParam("taskId") + taskId: String? + ): Result // { "startUser" : "启动人", "debug" : "true/false 是否调试版本", "status" : RUNNING } +} diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/service/ServiceBuildResource.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/service/ServiceBuildResource.kt index 7af7e664883..18d6be05fca 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/service/ServiceBuildResource.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/service/ServiceBuildResource.kt @@ -504,7 +504,13 @@ interface ServiceBuildResource { archiveFlag: Boolean? = false, @Parameter(description = "查看指定版本调试数据", required = false, example = "false") @QueryParam("version") - debugVersion: Int? = null + customVersion: Int? = null, + @Parameter(description = "触发代码库", required = false) + @QueryParam("triggerAlias") + triggerAlias: List? = null, + @Parameter(description = "触发分支", required = false) + @QueryParam("triggerBranch") + triggerBranch: List? = null ): Result> @Operation(summary = "获取构建详情") diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/service/ServicePipelineResource.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/service/ServicePipelineResource.kt index fc9c65ddc77..30a754c4912 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/service/ServicePipelineResource.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/service/ServicePipelineResource.kt @@ -31,11 +31,13 @@ import com.tencent.devops.common.api.auth.AUTH_HEADER_PROJECT_ID import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID_DEFAULT_VALUE import com.tencent.devops.common.api.pojo.Page -import com.tencent.devops.common.event.pojo.measure.PipelineLabelRelateInfo import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.common.event.pojo.measure.PipelineLabelRelateInfo import com.tencent.devops.common.pipeline.Model import com.tencent.devops.common.pipeline.ModelUpdate import com.tencent.devops.common.pipeline.enums.ChannelCode +import com.tencent.devops.common.pipeline.pojo.PipelineModelAndSetting +import com.tencent.devops.common.pipeline.pojo.setting.PipelineSetting import com.tencent.devops.common.web.annotation.BkApiPermission import com.tencent.devops.common.web.constant.BkApiHandleType import com.tencent.devops.process.engine.pojo.PipelineInfo @@ -46,15 +48,13 @@ import com.tencent.devops.process.pojo.PipelineId import com.tencent.devops.process.pojo.PipelineIdAndName import com.tencent.devops.process.pojo.PipelineIdInfo import com.tencent.devops.process.pojo.PipelineName +import com.tencent.devops.process.pojo.PipelineRemoteToken import com.tencent.devops.process.pojo.classify.PipelineViewPipelinePage import com.tencent.devops.process.pojo.pipeline.DeployPipelineResult import com.tencent.devops.process.pojo.pipeline.SimplePipeline -import com.tencent.devops.common.pipeline.pojo.PipelineModelAndSetting -import com.tencent.devops.common.pipeline.pojo.setting.PipelineSetting -import com.tencent.devops.process.pojo.PipelineRemoteToken -import io.swagger.v3.oas.annotations.tags.Tag import io.swagger.v3.oas.annotations.Operation import io.swagger.v3.oas.annotations.Parameter +import io.swagger.v3.oas.annotations.tags.Tag import javax.validation.Valid import javax.ws.rs.Consumes import javax.ws.rs.DELETE @@ -295,7 +295,7 @@ interface ServicePipelineResource { @Parameter(description = "流水线ID", required = true) @PathParam("pipelineId") pipelineId: String, - @Parameter(description = "渠道号，默认为BS", required = false) + @Parameter(description = "渠道号，不指定则为空", required = false) @QueryParam("channelCode") channelCode: ChannelCode? ): Result @@ -316,7 +316,11 @@ interface ServicePipelineResource { pipelineId: String, @Parameter(description = "渠道号，默认为BS", required = false) @QueryParam("channelCode") - channelCode: ChannelCode + channelCode: ChannelCode, + @Parameter(description = "是否检查权限", required = false) + @QueryParam("checkFlag") + @DefaultValue("true") + checkFlag: Boolean? = true ): Result @Operation(summary = "流水线编排列表") diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/service/ServiceSubPipelineResource.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/service/ServiceSubPipelineResource.kt index 542c4af2e03..b5cf93f5457 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/service/ServiceSubPipelineResource.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/service/ServiceSubPipelineResource.kt @@ -88,6 +88,9 @@ interface ServiceSubPipelineResource { @Parameter(description = "构建ID", required = true) @QueryParam("buildId") buildId: String, + @Parameter(description = "当前流水线执行次数", required = false) + @QueryParam("executeCount") + executeCount: Int?, @Parameter(description = "插件ID", required = true) @QueryParam("taskId") taskId: String, diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/user/UserBuildResource.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/user/UserBuildResource.kt index 4dd9df4d2e4..834f9122939 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/user/UserBuildResource.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/user/UserBuildResource.kt @@ -38,6 +38,7 @@ import com.tencent.devops.common.pipeline.pojo.BuildParameters import com.tencent.devops.common.pipeline.pojo.StageReviewRequest import com.tencent.devops.common.pipeline.pojo.element.Element import com.tencent.devops.common.web.annotation.BkField +import com.tencent.devops.process.enums.HistorySearchType import com.tencent.devops.process.pojo.BuildHistory import com.tencent.devops.process.pojo.BuildHistoryRemark import com.tencent.devops.process.pojo.BuildId @@ -400,13 +401,13 @@ interface UserBuildResource { @Parameter(description = "每页多少条", required = false, example = "20") @QueryParam("pageSize") pageSize: Int?, - @Parameter(description = "代码库别名", required = false) + @Parameter(description = "源材料代码库别名", required = false) @QueryParam("materialAlias") materialAlias: List?, @Parameter(description = "代码库URL", required = false) @QueryParam("materialUrl") materialUrl: String?, - @Parameter(description = "分支", required = false) + @Parameter(description = "源材料分支", required = false) @QueryParam("materialBranch") materialBranch: List?, @Parameter(description = "commitId", required = false) @@ -462,7 +463,13 @@ interface UserBuildResource { archiveFlag: Boolean? = false, @Parameter(description = "查看指定版本调试数据", required = false, example = "false") @QueryParam("version") - debugVersion: Int? = null + customVersion: Int? = null, + @Parameter(description = "触发代码库", required = false) + @QueryParam("triggerAlias") + triggerAlias: List?, + @Parameter(description = "触发分支", required = false) + @QueryParam("triggerBranch") + triggerBranch: List? ): Result> @Operation(summary = "修改流水线备注") @@ -534,7 +541,13 @@ interface UserBuildResource { pipelineId: String, @Parameter(description = "查看指定版本调试数据", required = false, example = "false") @QueryParam("version") - debugVersion: Int? = null + debugVersion: Int? = null, + @Parameter(description = "搜索分支关键字", required = false) + @QueryParam("search") + search: String?, + @Parameter(description = "搜索类型, 触发/源材料", required = false) + @QueryParam("type") + type: HistorySearchType? ): Result> @Operation(summary = "获取流水线构建中的查询条件-分支") @@ -556,7 +569,13 @@ interface UserBuildResource { alias: List?, @Parameter(description = "查看指定版本调试数据", required = false, example = "false") @QueryParam("debugVersion") - debugVersion: Int? = null + debugVersion: Int? = null, + @Parameter(description = "搜索分支关键字", required = false) + @QueryParam("search") + search: String?, + @Parameter(description = "搜索类型,触发/源材料", required = false) + @QueryParam("type") + type: HistorySearchType? ): Result> @Operation(summary = "触发审核") diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/user/UserPipelineVersionResource.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/user/UserPipelineVersionResource.kt index 829273663a4..560392ba7d1 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/user/UserPipelineVersionResource.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/user/UserPipelineVersionResource.kt @@ -35,7 +35,6 @@ import com.tencent.devops.common.pipeline.PipelineVersionWithModel import com.tencent.devops.common.pipeline.PipelineVersionWithModelRequest import com.tencent.devops.common.pipeline.pojo.TemplateInstanceCreateRequest import com.tencent.devops.common.pipeline.pojo.transfer.PreviewResponse -import com.tencent.devops.process.engine.pojo.PipelineVersionWithInfo import com.tencent.devops.process.pojo.PipelineDetail import com.tencent.devops.process.pojo.PipelineOperationDetail import com.tencent.devops.process.pojo.PipelineVersionReleaseRequest @@ -237,7 +236,7 @@ interface UserPipelineVersionResource { @Parameter(description = "每页多少条", required = false, example = "5") @QueryParam("pageSize") pageSize: Int? - ): Result> + ): Result> @Operation(summary = "获取指定版本号的流水线编排版本信息") @GET @@ -255,7 +254,7 @@ interface UserPipelineVersionResource { @Parameter(description = "跳转定位的版本号", required = false) @PathParam("version") version: Int - ): Result + ): Result @Operation(summary = "获取流水线操作日志列表（分页）") @GET diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/user/UserSubPipelineInfoResource.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/user/UserSubPipelineInfoResource.kt index f81277c03cc..c1f9d252008 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/user/UserSubPipelineInfoResource.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/api/user/UserSubPipelineInfoResource.kt @@ -59,6 +59,12 @@ interface UserSubPipelineInfoResource { projectId: String, @Parameter(description = "流水线ID", required = false, example = "") @QueryParam("subPip") - pipelineId: String + pipelineId: String, + @Parameter(description = "是否包含常量", required = false, example = "") + @QueryParam("includeConst") + includeConst: Boolean? = true, + @Parameter(description = "是否包含非入参", required = false, example = "") + @QueryParam("includeNotRequired") + includeNotRequired: Boolean? = true ): Result> } diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/constant/PipelineBuildParamKey.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/constant/PipelineBuildParamKey.kt index e854431366a..67377418ca5 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/constant/PipelineBuildParamKey.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/constant/PipelineBuildParamKey.kt @@ -90,4 +90,5 @@ object PipelineBuildParamKey { const val CI_WORKSPACE = "ci.workspace" const val CI_FAILED_TASKNAMES = "ci.failed_tasknames" const val CI_FAILED_TASKS = "ci.failed_tasks" + const val CI_REMARK = "ci.remark" } diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/constant/ProcessMessageCode.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/constant/ProcessMessageCode.kt index de66ed9420e..032ca3425bd 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/constant/ProcessMessageCode.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/constant/ProcessMessageCode.kt @@ -351,10 +351,19 @@ object ProcessMessageCode { const val ERROR_TIMER_TRIGGER_SVN_BRANCH_NOT_EMPTY = "2101243" // 定时触发SVN分支不能为空 const val ERROR_PIPELINE_ELEMENT_CHECK_FAILED = "2101244" // 流水线有效性校验失败 + const val ERROR_TIMER_TRIGGER_REPO_NOT_FOUND = "2101245" // 定时触发代码库不存在 + const val ERROR_TIMER_TRIGGER_NEED_ENABLE_PAC = "2101246" // 定时触发需要流水线开启PAC + const val ERROR_PIPELINE_TIMER_BRANCH_IS_EMPTY = "2101247" // 流水线定时触发分支为空 + const val ERROR_PIPELINE_TIMER_BRANCH_NO_CHANGE = "2101248" // 定时触发分支{0}代码没有变更 + const val ERROR_PIPELINE_TIMER_BRANCH_NOT_FOUND = "2101249" // 定时触发分支{0}不存在 + const val ERROR_PIPELINE_TIMER_BRANCH_UNKNOWN = "2101252" // 定时触发分支{0}未知错误 const val ERROR_PIPELINE_JOB_ID_FORMAT = "2101250" // 流水线Job:{0}的jobId为空或长度超过{1}位 const val ERROR_PIPELINE_JOB_CONTROL_NODECURR = "2101251" // 流水线Job:{0}的单节点或总结点并发配置需要为小于1000的正整数 + const val ERROR_PIPELINE_CONDITION_EXPRESSION_TOO_LONG = "2101253" // 自定义条件表达式{0}的长度超过{1}位 + const val ERROR_PIPELINE_BUILD_START_PARAM_NO_EMPTY = "2101254" // 构建启动参数如果必填，不能为空 + const val BK_SUCCESSFULLY_DISTRIBUTED = "bkSuccessfullyDistributed" // 跨项目构件分发成功，共分发了{0}个文件 const val BK_SUCCESSFULLY_FAILED = "bkSuccessfullyFailed" // 跨项目构件分发失败， const val BK_NO_MATCH_FILE_DISTRIBUTE = "bkNoMatchFileDistribute" // 匹配不到待分发的文件: {0} @@ -528,12 +537,17 @@ object ProcessMessageCode { const val BK_QUALITY_IN = "bkQualityIn" // 质量红线(准入) const val BK_QUALITY_OUT = "bkQualityOut" // 质量红线(准出) const val BK_BUILD_FINISHED_AND_DENY_PAUSE = "bkBuildFinishedAndDenyPause" // 构建已结束，禁止暂停请求 + const val BK_PIPELINE_RUN_CONDITION_RESULT = "bkPipelineRunConditionResult" // 执行条件计算结果 + const val BK_PIPELINE_RUN_CONDITION_NOT_MATCH = "bkPipelineRunConditionNotMatch" // 执行条件为满足，将跳过 + const val BK_PIPELINE_RUN_CONDITION_WITH_ERROR = "bkPipelineRunConditionWithError" // 执行条件计算报错 - // TODO: AgentReuseMutex const val BK_AGENT_REUSE_MUTEX = "bkAgentReuseMutex" const val BK_AGENT_REUSE_MUTEX_AGENT_ID_NULL = "bkAgentReuseMutexAgentIdNull" const val BK_MERGE_YAML_CREATE_FILE_TITLE = "bkMergeYamlCreateFileTitle" // 新增流水线发布mr标题 const val BK_MERGE_YAML_UPDATE_FILE_TITLE = "bkMergeYamlUpdateFileTitle" // 修改流水线发布mr标题 + const val BK_BUILD_QUEUE_WAIT_FOR_CONCURRENCY = "bkBuildQueueWaitingForConcurrency" // 并发组配置的排队 + const val BK_BUILD_QUEUE_WAIT = "bkBuildQueueWaiting" // 并发配置的排队 + const val BK_BUILD_CANCEL_BY_CONCURRENCY = "bkBuildCancelByConcurrency" // 并发组配置的取消 const val BK_PIPELINE_ELEMENT_CHECK_FAILED_MESSAGE = "bkPipelineElementCheckFailedMessage" // 没有子流水线执行权限错误标题 diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/engine/pojo/PipelineBuildContainer.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/engine/pojo/PipelineBuildContainer.kt index 002bb285cf2..d5c1436ff12 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/engine/pojo/PipelineBuildContainer.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/engine/pojo/PipelineBuildContainer.kt @@ -37,6 +37,7 @@ data class PipelineBuildContainer( val stageId: String, val containerId: String, // 与seq id同值 val containerHashId: String?, // 与model中的container.containerHashId同值 + val jobId: String?, val matrixGroupFlag: Boolean?, val matrixGroupId: String?, val containerType: String, diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/engine/pojo/PipelineBuildTask.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/engine/pojo/PipelineBuildTask.kt index 0e02e613aea..d0a6d86155f 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/engine/pojo/PipelineBuildTask.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/engine/pojo/PipelineBuildTask.kt @@ -61,7 +61,8 @@ data class PipelineBuildTask( var errorMsg: String? = null, val atomCode: String? = null, val stepId: String? = null, - var totalTime: Long? = null + var totalTime: Long? = null, + val jobId: String? = null ) { fun getTaskParam(paramName: String): String { return if (taskParams[paramName] != null) { diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/engine/pojo/PipelineInfo.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/engine/pojo/PipelineInfo.kt index b12137e156f..48ae18cec97 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/engine/pojo/PipelineInfo.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/engine/pojo/PipelineInfo.kt @@ -87,5 +87,7 @@ data class PipelineInfo( @get:Schema(title = "最新流水线版本状态（如有任何发布版本则为发布版本）", required = false) var latestVersionStatus: VersionStatus? = VersionStatus.RELEASED, @get:Schema(title = "流水线权限", required = false) - var permissions: PipelinePermissions? = null + var permissions: PipelinePermissions? = null, + @get:Schema(title = "流水线被锁定，即禁用", required = false) + var locked: Boolean? = false ) diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/enums/HistorySearchType.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/enums/HistorySearchType.kt new file mode 100644 index 00000000000..10e00ed6517 --- /dev/null +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/enums/HistorySearchType.kt @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.process.enums + +/** + * 构建历史搜索类型 + */ +enum class HistorySearchType { + // 触发器 + TRIGGER, + + // 源材料 + MATERIAL +} diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/enums/VariableType.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/enums/VariableType.kt index 1926d56be1a..32c16489613 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/enums/VariableType.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/enums/VariableType.kt @@ -33,8 +33,10 @@ enum class VariableType(val hasPrefix: Boolean = false, val alisName: String = " BK_CI_START_TYPE, BK_CI_PROJECT_NAME, BK_CI_PIPELINE_NAME, + BK_CI_BUILD_URL, BK_CI_BUILD_ID, BK_CI_BUILD_NUM, + BK_CI_BUILD_NUM_ALIAS, BK_CI_BUILD_JOB_ID(alisName = "job.id"), BK_CI_BUILD_MSG, BK_CI_BUILD_TASK_ID(alisName = "step.id"), @@ -52,6 +54,7 @@ enum class VariableType(val hasPrefix: Boolean = false, val alisName: String = " BK_CI_START_CHANNEL, BK_CI_START_USER_ID, BK_CI_START_USER_NAME, + BK_CI_PARENT_PROJECT_ID, BK_CI_PARENT_PIPELINE_ID, BK_CI_PARENT_BUILD_ID, BK_CI_START_PIPELINE_USER_ID, @@ -62,6 +65,19 @@ enum class VariableType(val hasPrefix: Boolean = false, val alisName: String = " BK_CI_TASK_NAME(alisName = "step.name"), BK_CI_ATOM_NAME(alisName = "step.atom_name"), + // GIT拉取常量 + BK_CI_GIT_REPO_URL, + BK_CI_GIT_REPO_NAME, + BK_CI_GIT_REPO_ALIAS_NAME, + BK_CI_GIT_REPO_BRANCH, + BK_CI_GIT_REPO_TAG, + BK_CI_GIT_REPO_CODE_PATH, + BK_CI_GIT_REPO_LAST_COMMIT_ID, + BK_CI_GIT_REPO_HEAD_COMMIT_ID, + BK_CI_GIT_REPO_HEAD_COMMIT_COMMENT, + BK_CI_GIT_REPO_HEAD_COMMIT_AUTHOR, + BK_CI_GIT_REPO_HEAD_COMMIT_COMMITTER, + // GIT事件触发公共变量 BK_CI_REPO_WEBHOOK_REPO_TYPE, BK_CI_REPO_WEBHOOK_REPO_URL, diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/BuildBasicInfo.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/BuildBasicInfo.kt index 23a7fdeffaa..e1d646d8c32 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/BuildBasicInfo.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/BuildBasicInfo.kt @@ -27,6 +27,7 @@ package com.tencent.devops.process.pojo +import com.tencent.devops.common.pipeline.enums.BuildStatus import io.swagger.v3.oas.annotations.media.Schema @Schema(title = "构建模型-基础信息") @@ -38,5 +39,7 @@ data class BuildBasicInfo( @get:Schema(title = "流水线ID", required = true) val pipelineId: String, @get:Schema(title = "流水线版本", required = true) - val pipelineVersion: Int + val pipelineVersion: Int, + @get:Schema(title = "构建状态", required = false) + val status: BuildStatus? ) diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/PipelineDetail.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/PipelineDetail.kt index 896bb2be30d..b689006e170 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/PipelineDetail.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/PipelineDetail.kt @@ -51,6 +51,8 @@ data class PipelineDetail( val instanceFromTemplate: Boolean, @get:Schema(title = "当前模板的ID", required = false) var templateId: String?, + @get:Schema(title = "关联模板版本", required = false) + var templateVersion: Long?, @get:Schema(title = "草稿或最新的发布版本") val version: Int, @get:Schema(title = "草稿或最新的发布版本名称") @@ -61,9 +63,9 @@ data class PipelineDetail( val baseVersionStatus: VersionStatus?, @get:Schema(title = "基准版本的版本名称") val baseVersionName: String?, - @get:Schema(title = "草稿或最新的发布版本") + @get:Schema(title = "最新的发布版本，如果为空则说明没有过发布版本") val releaseVersion: Int?, - @get:Schema(title = "草稿或最新的发布版本名称") + @get:Schema(title = "最新的发布版本名称，如果为空则说明没有过发布版本") val releaseVersionName: String?, @get:Schema(title = "是否有编辑权限") val hasPermission: Boolean, @@ -88,5 +90,7 @@ data class PipelineDetail( @get:Schema(title = "流水线YAML信息", required = false) val yamlInfo: PipelineYamlVo?, @get:Schema(title = "yaml文件在默认分支是否存在", required = false) - var yamlExist: Boolean? = false + var yamlExist: Boolean? = false, + @get:Schema(title = "运行锁定", required = false) + val locked: Boolean ) diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/PipelineDetailInfo.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/PipelineDetailInfo.kt index c4ea497b904..23953f2d631 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/PipelineDetailInfo.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/PipelineDetailInfo.kt @@ -63,5 +63,7 @@ data class PipelineDetailInfo( @get:Schema(title = "最新流水线版本状态（如有任何发布版本则为发布版本）", required = false) var latestVersionStatus: VersionStatus? = VersionStatus.RELEASED, @get:Schema(title = "流水线组名称列表", required = false) - var viewNames: List? + var viewNames: List?, + @get:Schema(title = "运行锁定", required = false) + val locked: Boolean = false ) diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/app/StartBuildContext.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/app/StartBuildContext.kt index 5cd23c7344f..ca7b1b858c8 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/app/StartBuildContext.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/app/StartBuildContext.kt @@ -31,10 +31,8 @@ import com.tencent.devops.common.api.constant.coerceAtMaxLength import com.tencent.devops.common.api.util.EnvUtils import com.tencent.devops.common.api.util.Watcher import com.tencent.devops.common.event.enums.ActionType -import com.tencent.devops.common.pipeline.Model import com.tencent.devops.common.pipeline.container.Container import com.tencent.devops.common.pipeline.container.Stage -import com.tencent.devops.common.pipeline.container.TriggerContainer import com.tencent.devops.common.pipeline.enums.BuildStatus import com.tencent.devops.common.pipeline.enums.ChannelCode import com.tencent.devops.common.pipeline.enums.StartType @@ -42,6 +40,9 @@ import com.tencent.devops.common.pipeline.pojo.BuildNoType import com.tencent.devops.common.pipeline.pojo.BuildParameters import com.tencent.devops.common.pipeline.pojo.element.Element import com.tencent.devops.common.pipeline.pojo.element.trigger.enums.CodeType +import com.tencent.devops.common.pipeline.pojo.setting.PipelineRunLockType +import com.tencent.devops.common.pipeline.pojo.setting.PipelineSetting +import com.tencent.devops.common.pipeline.utils.PIPELINE_GIT_EVENT_URL import com.tencent.devops.common.webhook.pojo.code.BK_REPO_GIT_WEBHOOK_EVENT_TYPE import com.tencent.devops.common.webhook.pojo.code.BK_REPO_GIT_WEBHOOK_ISSUE_IID import com.tencent.devops.common.webhook.pojo.code.BK_REPO_GIT_WEBHOOK_MR_ID @@ -63,9 +64,6 @@ import com.tencent.devops.common.webhook.pojo.code.PIPELINE_WEBHOOK_EVENT_TYPE import com.tencent.devops.common.webhook.pojo.code.PIPELINE_WEBHOOK_REVISION import com.tencent.devops.common.webhook.pojo.code.PIPELINE_WEBHOOK_TYPE import com.tencent.devops.process.pojo.code.WebhookInfo -import com.tencent.devops.common.pipeline.pojo.setting.PipelineRunLockType -import com.tencent.devops.common.pipeline.pojo.setting.PipelineSetting -import com.tencent.devops.common.pipeline.utils.PIPELINE_GIT_EVENT_URL import com.tencent.devops.process.utils.BK_CI_MATERIAL_ID import com.tencent.devops.process.utils.BK_CI_MATERIAL_NAME import com.tencent.devops.process.utils.BK_CI_MATERIAL_URL @@ -89,8 +87,8 @@ import com.tencent.devops.process.utils.PIPELINE_START_USER_ID import com.tencent.devops.process.utils.PIPELINE_START_USER_NAME import com.tencent.devops.process.utils.PipelineVarUtil import com.tencent.devops.process.utils.PipelineVarUtil.CONTEXT_PREFIX -import org.slf4j.LoggerFactory import java.time.LocalDateTime +import org.slf4j.LoggerFactory /** * 启动流水线上下文类，属于非线程安全类 @@ -136,7 +134,7 @@ data class StartBuildContext( // 注意：该字段在 PipelineContainerService.setUpTriggerContainer 中可能会被修改 var currentBuildNo: Int? = null, val debug: Boolean, - val debugModel: Model? + val debugModelStr: String? ) { val watcher: Watcher = Watcher("startBuild-$buildId") @@ -220,6 +218,7 @@ data class StartBuildContext( } fun needRerunStage(stage: Stage): Boolean { + // 重试所在stage的后续stage都需要 return stage.finally || retryStartTaskId == null || stage.id!! == retryStartTaskId } @@ -239,9 +238,10 @@ data class StartBuildContext( resourceVersion: Int, versionName: String?, yamlVersion: String?, - model: Model, + modelStr: String, debug: Boolean, pipelineSetting: PipelineSetting? = null, + realStartParamKeys: List, pipelineParamMap: MutableMap, webHookStartParam: MutableMap = mutableMapOf(), triggerReviewers: List? = null, @@ -250,7 +250,6 @@ data class StartBuildContext( val params: Map = pipelineParamMap.values.associate { it.key to it.value.toString() } // 解析出定义的流水线变量 - val realStartParamKeys = (model.stages[0].containers[0] as TriggerContainer).params.map { it.id } val retryStartTaskId = params[PIPELINE_RETRY_START_TASK_ID] val (actionType, executeCount, isStageRetry) = if (params[PIPELINE_RETRY_COUNT] != null) { @@ -292,18 +291,15 @@ data class StartBuildContext( webhookInfo = getWebhookInfo(params), buildMsg = params[PIPELINE_BUILD_MSG]?.coerceAtMaxLength(MAX_LENGTH), buildParameters = genOriginStartParamsList(realStartParamKeys, pipelineParamMap), - // 优化并发组逻辑，只在正式执行且GROUP_LOCK时才保存进history表 - concurrencyGroup = if (!debug) { - pipelineSetting?.takeIf { it.runLockType == PipelineRunLockType.GROUP_LOCK } - ?.concurrencyGroup?.let { - val webhookParam = webHookStartParam.values.associate { p -> p.key to p.value.toString() } - val tConcurrencyGroup = EnvUtils.parseEnv( - it, PipelineVarUtil.fillContextVarMap(webhookParam.plus(params)) - ) - logger.info("[$pipelineId]|[$buildId]|ConcurrencyGroup=$tConcurrencyGroup") - tConcurrencyGroup - } - } else null, + concurrencyGroup = pipelineSetting?.takeIf { it.runLockType == PipelineRunLockType.GROUP_LOCK } + ?.concurrencyGroup?.let { + val webhookParam = webHookStartParam.values.associate { p -> p.key to p.value.toString() } + val tConcurrencyGroup = EnvUtils.parseEnv( + it, PipelineVarUtil.fillContextVarMap(webhookParam.plus(params)) + ) + logger.info("[$pipelineId]|[$buildId]|ConcurrencyGroup=$tConcurrencyGroup") + tConcurrencyGroup + }, triggerReviewers = triggerReviewers, startBuildStatus = if (triggerReviewers.isNullOrEmpty()) BuildStatus.QUEUE else BuildStatus.TRIGGER_REVIEWING, @@ -311,7 +307,7 @@ data class StartBuildContext( pipelineSetting = pipelineSetting, pipelineParamMap = pipelineParamMap, debug = debug, - debugModel = model, + debugModelStr = modelStr, yamlVersion = yamlVersion ) } @@ -378,7 +374,7 @@ data class StartBuildContext( * 是否支持自定义触发材料 */ private fun supportCustomMaterials(startType: String?) = startType == StartType.REMOTE.name || - startType == StartType.SERVICE.name + startType == StartType.SERVICE.name /** * 简易只为实现推送PipelineBuildStartEvent事件所需要的参数，不是全部 @@ -427,7 +423,7 @@ data class StartBuildContext( concurrencyGroup = null, pipelineSetting = null, debug = debug, - debugModel = null, + debugModelStr = null, yamlVersion = null ) diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/open/BuildStatusInfo.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/open/BuildStatusInfo.kt new file mode 100644 index 00000000000..f9ae8a0a365 --- /dev/null +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/open/BuildStatusInfo.kt @@ -0,0 +1,12 @@ +package com.tencent.devops.process.pojo.open + +import com.tencent.devops.common.pipeline.enums.BuildStatus + +/** + * open接口返回的构建状态封装 + */ +data class BuildStatusInfo( + val startUser: String, + val debug: Boolean, + val status: BuildStatus +) diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/pipeline/DeployPipelineResult.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/pipeline/DeployPipelineResult.kt index 5f74382451d..28231f6f075 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/pipeline/DeployPipelineResult.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/pipeline/DeployPipelineResult.kt @@ -42,5 +42,7 @@ data class DeployPipelineResult( @get:Schema(title = "生成版本名称", required = false) val versionName: String?, @get:Schema(title = "目标链接", required = false) - val targetUrl: String? = null + val targetUrl: String? = null, + @get:Schema(title = "yaml信息", required = false) + val yamlInfo: PipelineYamlVo? = null ) diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/pipeline/PipelineResourceVersion.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/pipeline/PipelineResourceVersion.kt index 7a903833fdc..14d745f56c0 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/pipeline/PipelineResourceVersion.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/pipeline/PipelineResourceVersion.kt @@ -48,12 +48,14 @@ data class PipelineResourceVersion( var yaml: String?, @get:Schema(title = "YAML编排版本", required = false) var yamlVersion: String?, - @get:Schema(title = "创建者", required = true) - val creator: String, @get:Schema(title = "版本名称", required = true) val versionName: String?, + @get:Schema(title = "创建者", required = true) + val creator: String, @get:Schema(title = "版本创建时间", required = true) val createTime: LocalDateTime, + @get:Schema(title = "更新操作人", required = true) + val updater: String?, @get:Schema(title = "版本修改时间", required = true) val updateTime: LocalDateTime?, @get:Schema(title = "发布版本号", required = false) diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/pipeline/PipelineYamlVo.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/pipeline/PipelineYamlVo.kt index a29556ee6ad..73edece4031 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/pipeline/PipelineYamlVo.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/pipeline/PipelineYamlVo.kt @@ -36,7 +36,7 @@ data class PipelineYamlVo( @get:Schema(title = "代码库hashId", required = true) val repoHashId: String, @get:Schema(title = "代码库类型", required = true) - val scmType: ScmType, + val scmType: ScmType? = null, @get:Schema(title = "yaml文件路径", required = true) val filePath: String, @get:Schema(title = "代码库项目路径", required = false) diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/pipeline/record/BuildRecordTask.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/pipeline/record/BuildRecordTask.kt index 79e5e2bbb1e..4ad156dbb90 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/pipeline/record/BuildRecordTask.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/pipeline/record/BuildRecordTask.kt @@ -74,7 +74,9 @@ data class BuildRecordTask( @get:Schema(title = "结束时间", required = true) var endTime: LocalDateTime? = null, @get:Schema(title = "业务时间戳集合", required = true) - var timestamps: Map + var timestamps: Map, + @get:Schema(title = "异步执行状态", required = true) + var asyncStatus: String? = null ) { companion object { fun MutableList.addRecords( diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/setting/PipelineVersionSimple.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/setting/PipelineVersionSimple.kt index c40c615e6f7..e2246515612 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/setting/PipelineVersionSimple.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/pojo/setting/PipelineVersionSimple.kt @@ -38,6 +38,8 @@ data class PipelineVersionSimple( val creator: String, @get:Schema(title = "创建时间戳", required = true) val createTime: Long, + @get:Schema(title = "更新操作人", required = true) + val updater: String?, @get:Schema(title = "更新时间戳", required = true) val updateTime: Long?, @get:Schema(title = "流水线版本号", required = true) @@ -65,5 +67,7 @@ data class PipelineVersionSimple( @get:Schema(title = "调试构建ID", required = false) val debugBuildId: String? = null, @get:Schema(title = "该版本的来源版本（空时一定为主路径）", required = false) - val baseVersion: Int? = null + val baseVersion: Int? = null, + @get:Schema(title = "当前最新正式版本标识", required = false) + var latestReleasedFlag: Boolean? = false ) diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/utils/Constants.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/utils/Constants.kt index 8e6a6b81e7e..40a3cdca327 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/utils/Constants.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/utils/Constants.kt @@ -34,6 +34,7 @@ const val PIPELINE_START_PARENT_PIPELINE_NAME = "BK_CI_PARENT_PIPELINE_NAME" const val PIPELINE_START_PARENT_BUILD_ID = "BK_CI_PARENT_BUILD_ID" // "pipeline.start.parent.build.id" const val PIPELINE_START_PARENT_BUILD_NUM = "BK_CI_PARENT_BUILD_NUM" const val PIPELINE_START_PARENT_BUILD_TASK_ID = "BK_CI_PARENT_BUILD_TASK_ID" // "pipeline.start.parent.build.task.id" +const val PIPELINE_START_PARENT_EXECUTE_COUNT = "BK_CI_PARENT_EXECUTE_COUNT" const val PIPELINE_START_USER_ID = "BK_CI_START_USER_ID" // "pipeline.start.user.id" const val PIPELINE_START_USER_NAME = "BK_CI_START_USER_NAME" // "pipeline.start.user.name" const val PIPELINE_START_WEBHOOK_USER_ID = "BK_CI_START_WEBHOOK_USER_ID" // "pipeline.start.webhook.user.id" @@ -52,6 +53,8 @@ const val PIPELINE_BUILD_NUM_ALIAS = "BK_CI_BUILD_NUM_ALIAS" const val PIPELINE_BUILD_URL = "BK_CI_BUILD_URL" // 禁用定时触发器参数,当流水线配置这个参数，并且值为true,则禁用定时触发器 const val PIPELINE_TIMER_DISABLE = "BK_CI_TIMER_DISABLE" +// 子流水线运行方式 +const val PIPELINE_START_SUB_RUN_MODE = "BK_CI_SUB_PIPELINE_RUN_MODE" const val GIT_MR_NUMBER = "BK_CI_GIT_MR_NUMBER" // git_mr_number const val GITHUB_PR_NUMBER = "BK_CI_GITHUB_PR_NUMBER" // github_pr_number @@ -208,6 +211,11 @@ const val PIPELINE_MATRIX_CON_RUNNING_SIZE_MAX = 20 */ const val PIPELINE_STAGE_CONTAINERS_COUNT_MAX = 256 +/** + * 流水线设置-自定义表达式执行条件的条件长度-最大值 + */ +const val PIPELINE_CONDITION_EXPRESSION_LENGTH_MAX = 512 + /** * 入库VAR表，流水线变量最大长度 */ diff --git a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/utils/PipelineVersionUtils.kt b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/utils/PipelineVersionUtils.kt index 1942b4f2681..481a6218aa8 100644 --- a/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/utils/PipelineVersionUtils.kt +++ b/src/backend/ci/core/process/api-process/src/main/kotlin/com/tencent/devops/process/utils/PipelineVersionUtils.kt @@ -28,9 +28,16 @@ package com.tencent.devops.process.utils import com.tencent.devops.common.pipeline.Model +import com.tencent.devops.common.pipeline.container.NormalContainer +import com.tencent.devops.common.pipeline.container.Stage import com.tencent.devops.common.pipeline.container.TriggerContainer +import com.tencent.devops.common.pipeline.container.VMBuildContainer +import com.tencent.devops.common.pipeline.pojo.element.Element import com.tencent.devops.process.pojo.setting.PipelineSettingVersion +import kotlin.reflect.full.declaredMemberProperties +import kotlin.reflect.jvm.isAccessible +@Suppress("ComplexMethod", "ComplexCondition") object PipelineVersionUtils { fun getVersionNameByModel( @@ -103,7 +110,7 @@ object PipelineVersionUtils { val newStages = newModel.stages.drop(1) val originParams = (originModel.stages.first().containers.first() as TriggerContainer).params val newParams = (newModel.stages.first().containers.first() as TriggerContainer).params - return if (originStages == newStages && originParams == newParams) { + return if (originStages.differ(newStages) && originParams == newParams) { currVersion } else { currVersion + 1 @@ -120,4 +127,78 @@ object PipelineVersionUtils { ): Int { return if (originSetting == newSetting) currVersion else currVersion + 1 } + + private fun List.differ(other: List): Boolean { + if (this != other && this.size != other.size) return false + this.forEachIndexed { sIndex, thisStage -> + val otherStage = other[sIndex] + if ( + thisStage != otherStage || thisStage.containers.size != otherStage.containers.size || + thisStage.checkIn != otherStage.checkIn || thisStage.checkOut != otherStage.checkOut || + thisStage.stageControlOption != otherStage.stageControlOption + ) { + return false + } + thisStage.containers.forEachIndexed { cIndex, thisContainer -> + val otherContainer = otherStage.containers[cIndex] + if (thisContainer != otherContainer && thisContainer.elements.size != otherContainer.elements.size) { + return false + } + if (thisContainer is VMBuildContainer && otherContainer is VMBuildContainer) { + if (thisContainer != otherContainer || thisContainer.dispatchType != otherContainer.dispatchType || + thisContainer.jobControlOption != otherContainer.jobControlOption + ) return false + } else if (thisContainer is NormalContainer && otherContainer is NormalContainer) { + if (thisContainer != otherContainer || + thisContainer.jobControlOption != otherContainer.jobControlOption + ) return false + } else { + return false + } + thisContainer.elements.forEachIndexed { eIndex, thisElement -> + val otherElement = otherContainer.elements[eIndex] + if (thisElement != otherElement) return false + if (thisElement.additionalOptions != otherElement.additionalOptions) return false + if (thisElement.differ(otherElement)) return false + } + } + } + return true + } + + fun Element.differ(other: Element): Boolean { + if (this::class != other::class) { + return true + } + + val v1Properties = this.javaClass.kotlin.declaredMemberProperties + val v2Properties = other.javaClass.kotlin.declaredMemberProperties + if (v1Properties.size != v2Properties.size) { + return true + } + + val v1Map = v1Properties.associate { + it.isAccessible = true + it.name to it.get(this) + } + + val v2Map = v2Properties.associate { + it.isAccessible = true + it.name to it.get(other) + } + + if (v1Map.size != v2Map.size) { + return true + } + + for ((key, value) in v1Map) { + if (!v2Map.containsKey(key)) { + return true + } + if (v2Map[key] != value) { + return true + } + } + return false + } } diff --git a/src/backend/ci/core/process/api-process/src/test/kotlin/com/tencent/devops/process/utils/PipelineYamlVersionUtilsTest.kt b/src/backend/ci/core/process/api-process/src/test/kotlin/com/tencent/devops/process/utils/PipelineYamlVersionUtilsTest.kt index a19e59db135..f070b9e2736 100644 --- a/src/backend/ci/core/process/api-process/src/test/kotlin/com/tencent/devops/process/utils/PipelineYamlVersionUtilsTest.kt +++ b/src/backend/ci/core/process/api-process/src/test/kotlin/com/tencent/devops/process/utils/PipelineYamlVersionUtilsTest.kt @@ -27,11 +27,18 @@ package com.tencent.devops.process.utils +import com.tencent.devops.common.api.util.JsonUtil import com.tencent.devops.common.pipeline.Model +import com.tencent.devops.common.pipeline.NameAndValue import com.tencent.devops.common.pipeline.container.NormalContainer import com.tencent.devops.common.pipeline.container.Stage import com.tencent.devops.common.pipeline.container.TriggerContainer +import com.tencent.devops.common.pipeline.container.VMBuildContainer import com.tencent.devops.common.pipeline.enums.BuildScriptType +import com.tencent.devops.common.pipeline.enums.StageRunCondition +import com.tencent.devops.common.pipeline.enums.VMBaseOS +import com.tencent.devops.common.pipeline.option.StageControlOption +import com.tencent.devops.common.pipeline.pojo.element.ElementAdditionalOptions import com.tencent.devops.common.pipeline.pojo.element.agent.LinuxScriptElement import com.tencent.devops.common.pipeline.pojo.element.trigger.ManualTriggerElement import com.tencent.devops.common.pipeline.pojo.setting.PipelineRunLockType @@ -41,6 +48,7 @@ import com.tencent.devops.process.pojo.setting.PipelineSettingVersion import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Test +@Suppress("ALL") class PipelineYamlVersionUtilsTest { @Test @@ -174,7 +182,46 @@ class PipelineYamlVersionUtilsTest { LinuxScriptElement( script = "echo 1", continueNoneZero = true, - scriptType = BuildScriptType.SHELL + scriptType = BuildScriptType.SHELL, + additionalOptions = ElementAdditionalOptions(enable = true) + ) + ) + ) + ) + ) + ), + pipelineCreator = "userId" + ) + val diffModel4 = Model( + name = "name1", + desc = "", + stages = listOf( + Stage( + id = "stage-1", + containers = listOf( + TriggerContainer( + id = "0", + name = "trigger", + elements = listOf( + ManualTriggerElement( + id = "T-1-1-1", + name = "t1" + ) + ) + ) + ) + ), + Stage( + id = "stage-2", + containers = listOf( + NormalContainer(), + NormalContainer( + elements = listOf( + LinuxScriptElement( + script = "echo 1", + continueNoneZero = true, + scriptType = BuildScriptType.SHELL, + additionalOptions = ElementAdditionalOptions(enable = false) ) ) ) @@ -212,6 +259,248 @@ class PipelineYamlVersionUtilsTest { assertEquals(version + 1, PipelineVersionUtils.getPipelineVersion(version, model, diffModel2)) assertEquals(version + 1, PipelineVersionUtils.getTriggerVersion(version, model, diffTrigger)) assertEquals(version + 1, PipelineVersionUtils.getPipelineVersion(version, diffModel2, diffModel3)) + assertEquals(version + 1, PipelineVersionUtils.getPipelineVersion(version, diffModel3, diffModel4)) + } + + @Test + fun testModelStr() { + val modelStr1 = "{\"name\":\"插件enable\",\"desc\":\"插件enable\",\"stages\":[{\"containers\":[{\"@type\":\"trigger\",\"id\":\"0\",\"name\":\"trigger\",\"elements\":[{\"@type\":\"manualTrigger\",\"name\":\"手动触发\",\"id\":\"T-1-1-1\",\"canElementSkip\":false,\"useLatestParameters\":false,\"executeCount\":1,\"version\":\"1.*\",\"classType\":\"manualTrigger\",\"elementEnable\":true,\"atomCode\":\"manualTrigger\",\"taskAtom\":\"\"}],\"params\":[],\"containerId\":\"0\",\"containerHashId\":\"c-9658289a5be3436dbc5c96ae4b66f3ec\",\"matrixGroupFlag\":false,\"classType\":\"trigger\",\"containerEnable\":true}],\"id\":\"stage-1\",\"name\":\"stage-1\",\"tag\":[\"28ee946a59f64949a74f3dee40a1bda4\"],\"fastKill\":false,\"finally\":false,\"stageEnable\":true},{\"containers\":[{\"@type\":\"normal\",\"id\":\"1\",\"name\":\"无编译环境\",\"elements\":[{\"@type\":\"marketBuildLess\",\"name\":\"子流水线调用新\",\"id\":\"e-53f57f458b5745c6b1d20a5a20726253\",\"atomCode\":\"SubPipelineExec\",\"version\":\"1.*\",\"data\":{\"input\":{\"projectId\":\"codecc-tool-auto\",\"subPipelineType\":\"ID\",\"subPip\":\"p-3d2d4022cf3e476bb7c060e3634abff4\",\"subPipelineName\":\"\",\"runMode\":\"syn\",\"params\":\"[{\\\"key\\\":\\\"BK_CI_BUILD_MSG\\\",\\\"value\\\":\\\"1\\\",\\\"enable\\\":true},{\\\"key\\\":\\\"CODECC_BKCHECK_MinHeapFreeRatio\\\",\\\"value\\\":\\\"30\\\",\\\"enable\\\":true},{\\\"key\\\":\\\"CODECC_BKCHECK_MaxHeapFreeRatio\\\",\\\"value\\\":\\\"50\\\",\\\"enable\\\":true}]\",\"fieldNamespace\":\"sub_pipeline\"},\"output\":{\"sub_pipeline_buildId\":\"string\",\"sub_pipeline_url\":\"string\"},\"namespace\":\"\"},\"executeCount\":1,\"additionalOptions\":{\"enable\":false,\"continueWhenFailed\":false,\"manualSkip\":false,\"retryWhenFailed\":false,\"retryCount\":1,\"manualRetry\":false,\"timeout\":900,\"timeoutVar\":\"900\",\"runCondition\":\"PRE_TASK_SUCCESS\",\"pauseBeforeExec\":false,\"subscriptionPauseUser\":\"user1\",\"otherTask\":\"\",\"customVariables\":[{\"key\":\"param1\",\"value\":\"\"}],\"customCondition\":\"\",\"enableCustomEnv\":true},\"classType\":\"marketBuildLess\",\"elementEnable\":false,\"taskAtom\":\"\"}],\"enableSkip\":false,\"containerId\":\"1\",\"containerHashId\":\"c-5c8c288bce3c417a80d74e7d578bdab7\",\"maxQueueMinutes\":60,\"maxRunningMinutes\":1440,\"jobControlOption\":{\"enable\":true,\"prepareTimeout\":10,\"timeout\":900,\"timeoutVar\":\"900\",\"runCondition\":\"STAGE_RUNNING\",\"customVariables\":[{\"key\":\"param1\",\"value\":\"\"}],\"customCondition\":\"\",\"dependOnType\":\"ID\",\"dependOnId\":[],\"dependOnName\":\"\",\"continueWhenFailed\":false},\"jobId\":\"job_fLR\",\"matrixGroupFlag\":false,\"classType\":\"normal\",\"containerEnable\":true}],\"id\":\"stage-2\",\"name\":\"stage-1\",\"tag\":[\"28ee946a59f64949a74f3dee40a1bda4\"],\"fastKill\":false,\"finally\":false,\"checkIn\":{\"manualTrigger\":false,\"timeout\":24,\"markdownContent\":false,\"notifyType\":[\"RTX\"]},\"checkOut\":{\"manualTrigger\":false,\"timeout\":24,\"markdownContent\":false,\"notifyType\":[\"RTX\"]},\"stageEnable\":true}],\"labels\":[],\"instanceFromTemplate\":false,\"pipelineCreator\":\"user1\",\"events\":{},\"staticViews\":[],\"latestVersion\":1}\n" + val modelStr2 = "{\"name\":\"插件enable\",\"desc\":\"插件enable\",\"stages\":[{\"containers\":[{\"@type\":\"trigger\",\"id\":\"0\",\"name\":\"trigger\",\"elements\":[{\"@type\":\"manualTrigger\",\"name\":\"手动触发\",\"id\":\"T-1-1-1\",\"canElementSkip\":false,\"useLatestParameters\":false,\"executeCount\":1,\"version\":\"1.*\",\"classType\":\"manualTrigger\",\"elementEnable\":true,\"atomCode\":\"manualTrigger\",\"taskAtom\":\"\"}],\"params\":[],\"containerId\":\"0\",\"containerHashId\":\"c-9658289a5be3436dbc5c96ae4b66f3ec\",\"matrixGroupFlag\":false,\"classType\":\"trigger\",\"containerEnable\":true}],\"id\":\"stage-1\",\"name\":\"stage-1\",\"tag\":[\"28ee946a59f64949a74f3dee40a1bda4\"],\"fastKill\":false,\"finally\":false,\"stageEnable\":true},{\"containers\":[{\"@type\":\"normal\",\"id\":\"1\",\"name\":\"无编译环境\",\"elements\":[{\"@type\":\"marketBuildLess\",\"name\":\"子流水线调用新\",\"id\":\"e-53f57f458b5745c6b1d20a5a20726253\",\"atomCode\":\"SubPipelineExec\",\"version\":\"1.*\",\"data\":{\"input\":{\"projectId\":\"codecc-tool-auto\",\"subPipelineType\":\"ID\",\"subPip\":\"p-3d2d4022cf3e476bb7c060e3634abff4\",\"subPipelineName\":\"\",\"runMode\":\"syn\",\"params\":\"[{\\\"key\\\":\\\"BK_CI_BUILD_MSG\\\",\\\"value\\\":\\\"1\\\",\\\"enable\\\":true},{\\\"key\\\":\\\"CODECC_BKCHECK_MinHeapFreeRatio\\\",\\\"value\\\":\\\"30\\\",\\\"enable\\\":true},{\\\"key\\\":\\\"CODECC_BKCHECK_MaxHeapFreeRatio\\\",\\\"value\\\":\\\"50\\\",\\\"enable\\\":true}]\",\"fieldNamespace\":\"sub_pipeline\"},\"output\":{\"sub_pipeline_buildId\":\"string\",\"sub_pipeline_url\":\"string\"},\"namespace\":\"\"},\"executeCount\":1,\"additionalOptions\":{\"enable\":true,\"continueWhenFailed\":false,\"manualSkip\":false,\"retryWhenFailed\":false,\"retryCount\":1,\"manualRetry\":false,\"timeout\":900,\"timeoutVar\":\"900\",\"runCondition\":\"PRE_TASK_SUCCESS\",\"pauseBeforeExec\":false,\"subscriptionPauseUser\":\"user1\",\"otherTask\":\"\",\"customVariables\":[{\"key\":\"param1\",\"value\":\"\"}],\"customCondition\":\"\",\"enableCustomEnv\":true},\"classType\":\"marketBuildLess\",\"elementEnable\":true,\"taskAtom\":\"\"}],\"enableSkip\":false,\"containerId\":\"1\",\"containerHashId\":\"c-5c8c288bce3c417a80d74e7d578bdab7\",\"maxQueueMinutes\":60,\"maxRunningMinutes\":1440,\"jobControlOption\":{\"enable\":true,\"prepareTimeout\":10,\"timeout\":900,\"timeoutVar\":\"900\",\"runCondition\":\"STAGE_RUNNING\",\"customVariables\":[{\"key\":\"param1\",\"value\":\"\"}],\"customCondition\":\"\",\"dependOnType\":\"ID\",\"dependOnId\":[],\"dependOnName\":\"\",\"continueWhenFailed\":false},\"jobId\":\"job_fLR\",\"matrixGroupFlag\":false,\"classType\":\"normal\",\"containerEnable\":true}],\"id\":\"stage-2\",\"name\":\"stage-1\",\"tag\":[\"28ee946a59f64949a74f3dee40a1bda4\"],\"fastKill\":false,\"finally\":false,\"checkIn\":{\"manualTrigger\":false,\"timeout\":24,\"markdownContent\":false,\"notifyType\":[\"RTX\"]},\"checkOut\":{\"manualTrigger\":false,\"timeout\":24,\"markdownContent\":false,\"notifyType\":[\"RTX\"]},\"stageEnable\":true}],\"labels\":[],\"instanceFromTemplate\":false,\"pipelineCreator\":\"user1\",\"events\":{},\"staticViews\":[],\"latestVersion\":2}" + val model1 = JsonUtil.to(modelStr1, Model::class.java) + val model2 = JsonUtil.to(modelStr2, Model::class.java) + println(model1) + println(model2) + val version = 1 + assertEquals(version + 1, PipelineVersionUtils.getPipelineVersion(version, model1, model2)) + } + + @Test + fun testEnvDiffer() { + val model1 = Model( + name = "name1", + desc = "", + stages = listOf( + Stage( + id = "stage-1", + containers = listOf( + TriggerContainer( + id = "0", + name = "trigger", + elements = listOf( + ManualTriggerElement( + id = "T-1-1-1", + name = "t1" + ) + ) + ) + ) + ), + Stage( + id = "stage-2", + containers = listOf( + VMBuildContainer( + customEnv = listOf( + NameAndValue("a", "1") + ), + baseOS = VMBaseOS.LINUX + ), + NormalContainer( + elements = listOf( + LinuxScriptElement( + script = "echo 1", + continueNoneZero = true, + scriptType = BuildScriptType.SHELL, + additionalOptions = ElementAdditionalOptions(enable = true) + ) + ) + ) + ) + ) + ), + pipelineCreator = "userId" + ) + val model2 = Model( + name = "name1", + desc = "", + stages = listOf( + Stage( + id = "stage-1", + containers = listOf( + TriggerContainer( + id = "0", + name = "trigger", + elements = listOf( + ManualTriggerElement( + id = "T-1-1-1", + name = "t1" + ) + ) + ) + ) + ), + Stage( + id = "stage-2", + containers = listOf( + VMBuildContainer( + customEnv = emptyList(), + baseOS = VMBaseOS.LINUX + ), + NormalContainer( + elements = listOf( + LinuxScriptElement( + script = "echo 1", + continueNoneZero = true, + scriptType = BuildScriptType.SHELL, + additionalOptions = ElementAdditionalOptions(enable = true) + ) + ) + ) + ) + ) + ), + pipelineCreator = "userId" + ) + val model3 = Model( + name = "name1", + desc = "", + stages = listOf( + Stage( + id = "stage-1", + containers = listOf( + TriggerContainer( + id = "0", + name = "trigger", + elements = listOf( + ManualTriggerElement( + id = "T-1-1-1", + name = "t1" + ) + ) + ) + ) + ), + Stage( + id = "stage-2", + containers = listOf( + VMBuildContainer( + customEnv = listOf( + NameAndValue("a", "1") + ), + baseOS = VMBaseOS.LINUX + ), + NormalContainer( + elements = listOf( + LinuxScriptElement( + customEnv = listOf( + NameAndValue("b", "2") + ), + script = "echo 1", + continueNoneZero = true, + scriptType = BuildScriptType.SHELL, + additionalOptions = ElementAdditionalOptions(enable = true) + ) + ) + ) + ) + ) + ), + pipelineCreator = "userId" + ) + val version = 1 + assertEquals(version + 1, PipelineVersionUtils.getPipelineVersion(version, model1, model2)) + assertEquals(version + 1, PipelineVersionUtils.getPipelineVersion(version, model1, model3)) + } + + @Test + fun testRunConditionDiffer() { + val model1 = Model( + name = "name1", + desc = "", + stages = listOf( + Stage( + id = "stage-1", + containers = listOf( + TriggerContainer( + id = "0", + name = "trigger", + elements = listOf( + ManualTriggerElement( + id = "T-1-1-1", + name = "t1" + ) + ) + ) + ) + ), + Stage( + id = "stage-2", + containers = listOf( + VMBuildContainer( + baseOS = VMBaseOS.LINUX + ), + NormalContainer( + elements = listOf( + LinuxScriptElement( + script = "echo 1", + continueNoneZero = true, + scriptType = BuildScriptType.SHELL, + additionalOptions = ElementAdditionalOptions(enable = true) + ) + ) + ) + ), + stageControlOption = StageControlOption( + runCondition = StageRunCondition.AFTER_LAST_FINISHED + ) + ) + ), + pipelineCreator = "userId" + ) + val model2 = Model( + name = "name1", + desc = "", + stages = listOf( + Stage( + id = "stage-1", + containers = listOf( + TriggerContainer( + id = "0", + name = "trigger", + elements = listOf( + ManualTriggerElement( + id = "T-1-1-1", + name = "t1" + ) + ) + ) + ) + ), + Stage( + id = "stage-2", + containers = listOf( + VMBuildContainer( + baseOS = VMBaseOS.LINUX + ), + NormalContainer( + elements = listOf( + LinuxScriptElement( + script = "echo 1", + continueNoneZero = true, + scriptType = BuildScriptType.SHELL, + additionalOptions = ElementAdditionalOptions(enable = true) + ) + ) + ) + ), + stageControlOption = StageControlOption( + runCondition = StageRunCondition.CUSTOM_VARIABLE_MATCH + ) + ) + ), + pipelineCreator = "userId" + ) + val version = 1 + assertEquals(version + 1, PipelineVersionUtils.getPipelineVersion(version, model1, model2)) } @Test diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/PipelineSettingDao.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/PipelineSettingDao.kt index dfbbd82058a..59b2b2d950e 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/PipelineSettingDao.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/PipelineSettingDao.kt @@ -31,17 +31,12 @@ import com.fasterxml.jackson.core.type.TypeReference import com.tencent.devops.common.api.pojo.PipelineAsCodeSettings import com.tencent.devops.common.api.util.DateTimeUtil import com.tencent.devops.common.api.util.JsonUtil -import com.tencent.devops.common.notify.enums.NotifyType import com.tencent.devops.common.pipeline.pojo.setting.PipelineRunLockType import com.tencent.devops.common.pipeline.pojo.setting.PipelineSetting import com.tencent.devops.common.pipeline.pojo.setting.PipelineSubscriptionType import com.tencent.devops.common.pipeline.pojo.setting.Subscription import com.tencent.devops.model.process.tables.TPipelineSetting import com.tencent.devops.model.process.tables.records.TPipelineSettingRecord -import com.tencent.devops.process.utils.PIPELINE_RES_NUM_MIN -import com.tencent.devops.process.utils.PIPELINE_SETTING_MAX_QUEUE_SIZE_DEFAULT -import com.tencent.devops.process.utils.PIPELINE_SETTING_WAIT_QUEUE_TIME_MINUTE_DEFAULT -import com.tencent.devops.process.yaml.utils.NotifyTemplateUtils import org.jooq.Condition import org.jooq.DSLContext import org.jooq.Record1 @@ -54,80 +49,6 @@ import org.springframework.stereotype.Repository @Repository class PipelineSettingDao { - // 新流水线创建的时候，设置默认的通知配置。 - fun insertNewSetting( - dslContext: DSLContext, - projectId: String, - pipelineId: String, - pipelineName: String, - isTemplate: Boolean = false, - successNotifyTypes: String = "", - failNotifyTypes: String = "${NotifyType.EMAIL.name},${NotifyType.RTX.name}", - maxPipelineResNum: Int? = PIPELINE_RES_NUM_MIN, - pipelineAsCodeSettings: PipelineAsCodeSettings?, - settingVersion: Int - ): PipelineSetting? { - with(TPipelineSetting.T_PIPELINE_SETTING) { - val failType = failNotifyTypes.split(",").filter { i -> i.isNotBlank() } - .map { type -> PipelineSubscriptionType.valueOf(type) }.toSet() - val failSubscription = Subscription( - types = failType, - groups = emptySet(), - users = "\${{ci.actor}}", - content = NotifyTemplateUtils.getCommonShutdownFailureContent() - ) - val result = dslContext.insertInto( - this, - PROJECT_ID, - PIPELINE_ID, - NAME, - RUN_LOCK_TYPE, - DESC, - SUCCESS_RECEIVER, - FAIL_RECEIVER, - SUCCESS_GROUP, - FAIL_GROUP, - SUCCESS_TYPE, - FAIL_TYPE, - SUCCESS_CONTENT, - FAIL_CONTENT, - WAIT_QUEUE_TIME_SECOND, - MAX_QUEUE_SIZE, - IS_TEMPLATE, - MAX_PIPELINE_RES_NUM, - PIPELINE_AS_CODE_SETTINGS, - SUCCESS_SUBSCRIPTION, - FAILURE_SUBSCRIPTION, - VERSION - ).values( - projectId, - pipelineId, - pipelineName, - PipelineRunLockType.toValue(PipelineRunLockType.MULTIPLE), - "", - "", - failSubscription.users, - "", - "", - successNotifyTypes, - failNotifyTypes, - "", - failSubscription.content, - DateTimeUtil.minuteToSecond(PIPELINE_SETTING_WAIT_QUEUE_TIME_MINUTE_DEFAULT), - PIPELINE_SETTING_MAX_QUEUE_SIZE_DEFAULT, - isTemplate, - maxPipelineResNum, - pipelineAsCodeSettings?.let { self -> - JsonUtil.toJson(self, false) - }, - JsonUtil.toJson(listOf(), false), - JsonUtil.toJson(listOf(failSubscription), false), - settingVersion - ).returning().fetchOne() - return mapper.map(result) - } - } - fun saveSetting( dslContext: DSLContext, setting: PipelineSetting, @@ -178,22 +99,22 @@ class PipelineSettingDao { setting.desc, PipelineRunLockType.toValue(setting.runLockType), setting.pipelineId, - setting.successSubscription.users, - setting.failSubscription.users, - setting.successSubscription.groups.joinToString(","), - setting.failSubscription.groups.joinToString(","), - setting.successSubscription.types.joinToString(",") { it.name }, - setting.failSubscription.types.joinToString(",") { it.name }, - setting.failSubscription.wechatGroupFlag, - setting.failSubscription.wechatGroup, - setting.failSubscription.wechatGroupMarkdownFlag, - setting.successSubscription.wechatGroupFlag, - setting.successSubscription.wechatGroup, - setting.successSubscription.wechatGroupMarkdownFlag, - setting.successSubscription.detailFlag, - setting.failSubscription.detailFlag, - setting.successSubscription.content, - setting.failSubscription.content, + setting.successSubscription?.users, + setting.failSubscription?.users, + setting.successSubscription?.groups?.joinToString(","), + setting.failSubscription?.groups?.joinToString(","), + setting.successSubscription?.types?.joinToString(",") { it.name }, + setting.failSubscription?.types?.joinToString(",") { it.name }, + setting.failSubscription?.wechatGroupFlag ?: false, + setting.failSubscription?.wechatGroup ?: "", + setting.failSubscription?.wechatGroupMarkdownFlag ?: false, + setting.successSubscription?.wechatGroupFlag ?: false, + setting.successSubscription?.wechatGroup ?: "", + setting.successSubscription?.wechatGroupMarkdownFlag ?: false, + setting.successSubscription?.detailFlag ?: false, + setting.failSubscription?.detailFlag ?: false, + setting.successSubscription?.content, + setting.failSubscription?.content, DateTimeUtil.minuteToSecond(setting.waitQueueTimeMinute), setting.maxQueueSize, isTemplate, @@ -210,22 +131,22 @@ class PipelineSettingDao { .set(NAME, setting.pipelineName) .set(DESC, setting.desc) .set(RUN_LOCK_TYPE, PipelineRunLockType.toValue(setting.runLockType)) - .set(SUCCESS_RECEIVER, setting.successSubscription.users) - .set(FAIL_RECEIVER, setting.failSubscription.users) - .set(SUCCESS_GROUP, setting.successSubscription.groups.joinToString(",")) - .set(FAIL_GROUP, setting.failSubscription.groups.joinToString(",")) - .set(SUCCESS_TYPE, setting.successSubscription.types.joinToString(",") { it.name }) - .set(FAIL_TYPE, setting.failSubscription.types.joinToString(",") { it.name }) - .set(FAIL_WECHAT_GROUP_FLAG, setting.failSubscription.wechatGroupFlag) - .set(FAIL_WECHAT_GROUP, setting.failSubscription.wechatGroup) - .set(FAIL_WECHAT_GROUP_MARKDOWN_FLAG, setting.failSubscription.wechatGroupMarkdownFlag) - .set(SUCCESS_WECHAT_GROUP_FLAG, setting.successSubscription.wechatGroupFlag) - .set(SUCCESS_WECHAT_GROUP, setting.successSubscription.wechatGroup) - .set(SUCCESS_WECHAT_GROUP_MARKDOWN_FLAG, setting.successSubscription.wechatGroupMarkdownFlag) - .set(SUCCESS_DETAIL_FLAG, setting.successSubscription.detailFlag) - .set(FAIL_DETAIL_FLAG, setting.failSubscription.detailFlag) - .set(SUCCESS_CONTENT, setting.successSubscription.content) - .set(FAIL_CONTENT, setting.failSubscription.content) + .set(SUCCESS_RECEIVER, setting.successSubscription?.users) + .set(FAIL_RECEIVER, setting.failSubscription?.users) + .set(SUCCESS_GROUP, setting.successSubscription?.groups?.joinToString(",")) + .set(FAIL_GROUP, setting.failSubscription?.groups?.joinToString(",")) + .set(SUCCESS_TYPE, setting.successSubscription?.types?.joinToString(",") { it.name }) + .set(FAIL_TYPE, setting.failSubscription?.types?.joinToString(",") { it.name }) + .set(FAIL_WECHAT_GROUP_FLAG, setting.failSubscription?.wechatGroupFlag ?: false) + .set(FAIL_WECHAT_GROUP, setting.failSubscription?.wechatGroup ?: "") + .set(FAIL_WECHAT_GROUP_MARKDOWN_FLAG, setting.failSubscription?.wechatGroupMarkdownFlag ?: false) + .set(SUCCESS_WECHAT_GROUP_FLAG, setting.successSubscription?.wechatGroupFlag ?: false) + .set(SUCCESS_WECHAT_GROUP, setting.successSubscription?.wechatGroup ?: "") + .set(SUCCESS_WECHAT_GROUP_MARKDOWN_FLAG, setting.successSubscription?.wechatGroupMarkdownFlag ?: false) + .set(SUCCESS_DETAIL_FLAG, setting.successSubscription?.detailFlag ?: false) + .set(FAIL_DETAIL_FLAG, setting.failSubscription?.detailFlag ?: false) + .set(SUCCESS_CONTENT, setting.successSubscription?.content) + .set(FAIL_CONTENT, setting.failSubscription?.content) .set(WAIT_QUEUE_TIME_SECOND, DateTimeUtil.minuteToSecond(setting.waitQueueTimeMinute)) .set(MAX_QUEUE_SIZE, setting.maxQueueSize) .set(MAX_PIPELINE_RES_NUM, setting.maxPipelineResNum) @@ -433,17 +354,24 @@ class PipelineSettingDao { override fun map(record: TPipelineSettingRecord?): PipelineSetting? { return record?.let { t -> val successType = t.successType?.split(",")?.filter { i -> i.isNotBlank() } - ?.map { type -> PipelineSubscriptionType.valueOf(type) }?.toSet() ?: emptySet() + ?.map { type -> PipelineSubscriptionType.valueOf(type) }?.toMutableSet() ?: mutableSetOf() + // 老数据兼容,老数据的启用企业微信群通知,转换成微信组通知类型 + if (t.successWechatGroupFlag == true) { + successType.add(PipelineSubscriptionType.WEWORK_GROUP) + } val failType = t.failType?.split(",")?.filter { i -> i.isNotBlank() } - ?.map { type -> PipelineSubscriptionType.valueOf(type) }?.toSet() ?: emptySet() + ?.map { type -> PipelineSubscriptionType.valueOf(type) }?.toMutableSet() ?: mutableSetOf() + if (t.failWechatGroupFlag == true) { + failType.add(PipelineSubscriptionType.WEWORK_GROUP) + } var oldSuccessSubscription = Subscription( types = successType, groups = t.successGroup?.split(",")?.toSet() ?: emptySet(), users = t.successReceiver ?: "", wechatGroupFlag = t.successWechatGroupFlag ?: false, wechatGroup = t.successWechatGroup ?: "", - wechatGroupMarkdownFlag = t.successWechatGroupMarkdownFlag, - detailFlag = t.successDetailFlag, + wechatGroupMarkdownFlag = t.successWechatGroupMarkdownFlag ?: false, + detailFlag = t.successDetailFlag ?: false, content = t.successContent ?: "" ).takeIf { successType.isNotEmpty() } var oldFailSubscription = Subscription( @@ -453,7 +381,7 @@ class PipelineSettingDao { wechatGroupFlag = t.failWechatGroupFlag ?: false, wechatGroup = t.failWechatGroup ?: "", wechatGroupMarkdownFlag = t.failWechatGroupMarkdownFlag ?: false, - detailFlag = t.failDetailFlag, + detailFlag = t.failDetailFlag ?: false, content = t.failContent ?: "" ).takeIf { failType.isNotEmpty() } // 如果新数组有值，则老数据被替换为新数据的首个元素 @@ -461,14 +389,14 @@ class PipelineSettingDao { val list = JsonUtil.to(it, object : TypeReference>() {}) if (list.isNotEmpty()) { oldSuccessSubscription = list.first() - list + list.map { s -> s.fixWeworkGroupType() } } else null } ?: oldSuccessSubscription?.let { listOf(it) } val failSubscriptionList = t.failureSubscription?.let { val list = JsonUtil.to(it, object : TypeReference>() {}) if (list.isNotEmpty()) { oldFailSubscription = list.first() - list + list.map { s -> s.fixWeworkGroupType() } } else null } ?: oldFailSubscription?.let { listOf(it) } PipelineSetting( diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/PipelineSettingVersionDao.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/PipelineSettingVersionDao.kt index b914748f01b..0debcb1618d 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/PipelineSettingVersionDao.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/PipelineSettingVersionDao.kt @@ -205,9 +205,11 @@ class PipelineSettingVersionDao { runLockType = t.runLockType?.let { PipelineRunLockType.valueOf(it) }, successSubscriptionList = t.successSubscription?.let { JsonUtil.to(it, object : TypeReference>() {}) + .map { s -> s.fixWeworkGroupType() } }, failSubscriptionList = t.failureSubscription?.let { JsonUtil.to(it, object : TypeReference>() {}) + .map { s -> s.fixWeworkGroupType() } }, version = t.version, labels = t.labels?.let { self -> diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/record/BuildRecordTaskDao.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/record/BuildRecordTaskDao.kt index 277aceea870..7104801792a 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/record/BuildRecordTaskDao.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/dao/record/BuildRecordTaskDao.kt @@ -40,7 +40,7 @@ import com.tencent.devops.process.pojo.KEY_TASK_ID import com.tencent.devops.process.pojo.pipeline.record.BuildRecordTask import org.jooq.Condition import org.jooq.DSLContext -import org.jooq.Record18 +import org.jooq.Record19 import org.jooq.RecordMapper import org.jooq.impl.DSL import org.jooq.util.mysql.MySQLDSL @@ -211,7 +211,7 @@ class BuildRecordTaskDao { val result = dslContext.select( BUILD_ID, PROJECT_ID, PIPELINE_ID, RESOURCE_VERSION, STAGE_ID, CONTAINER_ID, TASK_ID, TASK_SEQ, EXECUTE_COUNT, TASK_VAR, CLASS_TYPE, ATOM_CODE, STATUS, ORIGIN_CLASS_TYPE, - START_TIME, END_TIME, TIMESTAMPS, POST_INFO + START_TIME, END_TIME, TIMESTAMPS, POST_INFO, ASYNC_STATUS ).from(this).join(max).on( TASK_ID.eq(max.field(KEY_TASK_ID, String::class.java)) .and(EXECUTE_COUNT.eq(max.field(KEY_EXECUTE_COUNT, Int::class.java))) @@ -238,7 +238,7 @@ class BuildRecordTaskDao { val result = dslContext.select( BUILD_ID, PROJECT_ID, PIPELINE_ID, RESOURCE_VERSION, STAGE_ID, CONTAINER_ID, TASK_ID, TASK_SEQ, EXECUTE_COUNT, TASK_VAR, CLASS_TYPE, ATOM_CODE, STATUS, ORIGIN_CLASS_TYPE, - START_TIME, END_TIME, TIMESTAMPS, POST_INFO + START_TIME, END_TIME, TIMESTAMPS, POST_INFO, ASYNC_STATUS ).from(this).where(conditions).orderBy(TASK_SEQ.asc()).fetch() return result.map { record -> generateBuildRecordTask(record) @@ -247,9 +247,9 @@ class BuildRecordTaskDao { } private fun TPipelineBuildRecordTask.generateBuildRecordTask( - record: Record18 + record: Record19 ) = BuildRecordTask( buildId = record[BUILD_ID], @@ -275,7 +275,8 @@ class BuildRecordTaskDao { } ?: mapOf(), elementPostInfo = record[POST_INFO]?.let { JsonUtil.to(it, object : TypeReference() {}) - } + }, + asyncStatus = record[ASYNC_STATUS] ) fun getRecord( @@ -298,6 +299,28 @@ class BuildRecordTaskDao { } } + fun updateAsyncStatus( + dslContext: DSLContext, + projectId: String, + pipelineId: String, + buildId: String, + taskId: String, + executeCount: Int, + asyncStatus: String + ) { + with(TPipelineBuildRecordTask.T_PIPELINE_BUILD_RECORD_TASK) { + dslContext.update(this) + .set(ASYNC_STATUS, asyncStatus) + .where( + BUILD_ID.eq(buildId) + .and(PROJECT_ID.eq(projectId)) + .and(PIPELINE_ID.eq(pipelineId)) + .and(TASK_ID.eq(taskId)) + .and(EXECUTE_COUNT.eq(executeCount)) + ).execute() + } + } + class BuildRecordTaskJooqMapper : RecordMapper { override fun map(record: TPipelineBuildRecordTaskRecord?): BuildRecordTask? { return record?.run { diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/atom/plugin/MarketBuildAtomElementBizPlugin.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/atom/plugin/MarketBuildAtomElementBizPlugin.kt index 45d3b144790..8db4306a8e1 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/atom/plugin/MarketBuildAtomElementBizPlugin.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/atom/plugin/MarketBuildAtomElementBizPlugin.kt @@ -35,6 +35,7 @@ import com.tencent.devops.common.pipeline.pojo.element.atom.ElementCheckResult import com.tencent.devops.common.pipeline.pojo.element.market.MarketBuildAtomElement import com.tencent.devops.process.plugin.ElementBizPlugin import com.tencent.devops.process.plugin.annotation.ElementBiz +import com.tencent.devops.process.pojo.pipeline.PipelineYamlVo import org.springframework.beans.factory.annotation.Autowired @ElementBiz @@ -54,7 +55,8 @@ class MarketBuildAtomElementBizPlugin @Autowired constructor( userId: String, channelCode: ChannelCode, create: Boolean, - container: Container + container: Container, + yamlInfo: PipelineYamlVo? ) = Unit override fun beforeDelete(element: MarketBuildAtomElement, param: BeforeDeleteParam) { diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/atom/plugin/MarketBuildLessAtomElementBizPlugin.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/atom/plugin/MarketBuildLessAtomElementBizPlugin.kt index de0741d12f2..d2654909f00 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/atom/plugin/MarketBuildLessAtomElementBizPlugin.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/atom/plugin/MarketBuildLessAtomElementBizPlugin.kt @@ -35,6 +35,7 @@ import com.tencent.devops.common.pipeline.pojo.element.atom.ElementCheckResult import com.tencent.devops.common.pipeline.pojo.element.market.MarketBuildLessAtomElement import com.tencent.devops.process.plugin.ElementBizPlugin import com.tencent.devops.process.plugin.annotation.ElementBiz +import com.tencent.devops.process.pojo.pipeline.PipelineYamlVo import org.springframework.beans.factory.annotation.Autowired @ElementBiz @@ -54,7 +55,8 @@ class MarketBuildLessAtomElementBizPlugin @Autowired constructor( userId: String, channelCode: ChannelCode, create: Boolean, - container: Container + container: Container, + yamlInfo: PipelineYamlVo? ) = Unit override fun beforeDelete(element: MarketBuildLessAtomElement, param: BeforeDeleteParam) { diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/atom/plugin/SubPipelineCallElementBizPlugin.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/atom/plugin/SubPipelineCallElementBizPlugin.kt index 507b463d0ed..4394f90322a 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/atom/plugin/SubPipelineCallElementBizPlugin.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/atom/plugin/SubPipelineCallElementBizPlugin.kt @@ -35,6 +35,7 @@ import com.tencent.devops.common.pipeline.pojo.element.atom.BeforeDeleteParam import com.tencent.devops.common.pipeline.pojo.element.atom.ElementCheckResult import com.tencent.devops.process.plugin.ElementBizPlugin import com.tencent.devops.process.plugin.annotation.ElementBiz +import com.tencent.devops.process.pojo.pipeline.PipelineYamlVo import org.springframework.beans.factory.annotation.Autowired @ElementBiz @@ -80,6 +81,7 @@ class SubPipelineCallElementBizPlugin @Autowired constructor( userId: String, channelCode: ChannelCode, create: Boolean, - container: Container + container: Container, + yamlInfo: PipelineYamlVo? ) = Unit } diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/compatibility/BuildParametersCompatibilityTransformer.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/compatibility/BuildParametersCompatibilityTransformer.kt index 29a6657475f..4b9e2b16bc9 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/compatibility/BuildParametersCompatibilityTransformer.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/compatibility/BuildParametersCompatibilityTransformer.kt @@ -41,6 +41,9 @@ interface BuildParametersCompatibilityTransformer { * 前端传入的为正确的新变量 */ fun parseTriggerParam( + userId: String, + projectId: String, + pipelineId: String, paramProperties: List, paramValues: Map ): MutableMap diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/compatibility/v2/V2BuildParametersCompatibilityTransformer.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/compatibility/v2/V2BuildParametersCompatibilityTransformer.kt index c5e31514630..195c185aad1 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/compatibility/v2/V2BuildParametersCompatibilityTransformer.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/compatibility/v2/V2BuildParametersCompatibilityTransformer.kt @@ -27,14 +27,20 @@ package com.tencent.devops.process.engine.compatibility.v2 +import com.tencent.devops.common.api.exception.ErrorCodeException import com.tencent.devops.common.pipeline.pojo.BuildFormProperty import com.tencent.devops.common.pipeline.pojo.BuildParameters +import com.tencent.devops.process.constant.ProcessMessageCode import com.tencent.devops.process.engine.compatibility.BuildParametersCompatibilityTransformer import com.tencent.devops.process.utils.PipelineVarUtil +import org.slf4j.LoggerFactory open class V2BuildParametersCompatibilityTransformer : BuildParametersCompatibilityTransformer { override fun parseTriggerParam( + userId: String, + projectId: String, + pipelineId: String, paramProperties: List, paramValues: Map ): MutableMap { @@ -42,16 +48,33 @@ open class V2BuildParametersCompatibilityTransformer : BuildParametersCompatibil val paramsMap = HashMap(paramProperties.size, 1F) paramProperties.forEach { param -> - // 通过对现有Model存在的旧变量替换成新变量， 如果已经是新的会为空，直接为it.id val key = PipelineVarUtil.oldVarToNewVar(param.id) ?: param.id // 现有用户覆盖定义旧系统变量的，前端无法帮助转换，用户传的仍然是旧变量为key，则用新的Key无法找到，要用旧的id兜底 // 如果编排中指定为常量，则必须以编排的默认值为准，不支持触发时传参覆盖 val value = if (param.constant == true) { + // 常量需要在启动是强制设为只读 + param.readOnly = true param.defaultValue +// } else if (!param.required) { +// // TODO #8161 没有作为前端可填入参的变量，直接取默认值，不可被覆盖（实施前仅打印日志） +// param.defaultValue } else { - paramValues[key] ?: paramValues[param.id] ?: param.defaultValue + val overrideValue = paramValues[key] ?: paramValues[param.id] + if (!param.required && overrideValue != null) { + logger.warn( + "BKSystemErrorMonitor|parseTriggerParam|$userId|$projectId|$pipelineId|[$key] " + + "not required, overrideValue=$overrideValue, defaultValue=${param.defaultValue}" + ) + } + overrideValue ?: param.defaultValue + } + if (param.valueNotEmpty == true && value.toString().isEmpty()) { + throw ErrorCodeException( + errorCode = ProcessMessageCode.ERROR_PIPELINE_BUILD_START_PARAM_NO_EMPTY, + params = arrayOf(param.id) + ) } paramsMap[key] = BuildParameters( @@ -66,4 +89,8 @@ open class V2BuildParametersCompatibilityTransformer : BuildParametersCompatibil return paramsMap } + + companion object { + private val logger = LoggerFactory.getLogger(V2BuildParametersCompatibilityTransformer::class.java) + } } diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/control/ControlUtils.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/control/ControlUtils.kt index 74d600e7bf9..5c2adde5aae 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/control/ControlUtils.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/control/ControlUtils.kt @@ -27,12 +27,10 @@ package com.tencent.devops.process.engine.control -import com.tencent.devops.common.api.expression.EvalExpress import com.tencent.devops.common.api.util.EnvUtils -import com.tencent.devops.common.expression.ExpressionParseException import com.tencent.devops.common.expression.ExpressionParser import com.tencent.devops.common.expression.expression.EvaluationResult -import com.tencent.devops.common.expression.expression.ParseExceptionKind +import com.tencent.devops.common.pipeline.EnvReplacementParser import com.tencent.devops.common.pipeline.NameAndValue import com.tencent.devops.common.pipeline.enums.BuildStatus import com.tencent.devops.common.pipeline.enums.JobRunCondition @@ -40,6 +38,7 @@ import com.tencent.devops.common.pipeline.enums.StageRunCondition import com.tencent.devops.common.pipeline.pojo.element.ElementAdditionalOptions import com.tencent.devops.common.pipeline.pojo.element.RunCondition import com.tencent.devops.common.web.utils.I18nUtil +import com.tencent.devops.process.constant.ProcessMessageCode import com.tencent.devops.process.constant.ProcessMessageCode.BK_CHECK_JOB_RUN_CONDITION import com.tencent.devops.process.constant.ProcessMessageCode.BK_CHECK_TASK_RUN_CONDITION import com.tencent.devops.process.constant.ProcessMessageCode.BK_CUSTOM_VARIABLES_ARE_ALL_SATISFIED @@ -51,6 +50,7 @@ import com.tencent.devops.process.constant.ProcessMessageCode.BK_TASK_DISABLED import com.tencent.devops.process.constant.ProcessMessageCode.BK_WHEN_THE_CUSTOM_VARIABLES_ARE_ALL_SATISFIED import com.tencent.devops.process.engine.pojo.PipelineBuildContainer import com.tencent.devops.process.util.TaskUtils +import com.tencent.devops.process.utils.PipelineVarUtil import com.tencent.devops.process.utils.TASK_FAIL_RETRY_MAX_COUNT import com.tencent.devops.process.utils.TASK_FAIL_RETRY_MIN_COUNT import org.slf4j.LoggerFactory @@ -171,8 +171,7 @@ object ControlUtils { containerFinalStatus: BuildStatus, variables: Map, hasFailedTaskInSuccessContainer: Boolean, - message: StringBuilder = StringBuilder(), - asCodeEnabled: Boolean + message: StringBuilder = StringBuilder() ): Boolean { message.append( I18nUtil.getCodeLanMessage(BK_CHECK_TASK_RUN_CONDITION) @@ -222,8 +221,7 @@ object ControlUtils { buildId = buildId, additionalOptions = additionalOptions, variables = variables, - message = message, - asCodeEnabled = asCodeEnabled + message = message ) } else -> { message.clear() @@ -237,17 +235,12 @@ object ControlUtils { buildId: String, additionalOptions: ElementAdditionalOptions?, variables: Map, - message: StringBuilder, - asCodeEnabled: Boolean + message: StringBuilder ): Boolean { if (additionalOptions?.runCondition == RunCondition.CUSTOM_CONDITION_MATCH && !additionalOptions.customCondition.isNullOrBlank() ) { - return if (asCodeEnabled) { - !evalExpressionAsCode(additionalOptions.customCondition, buildId, variables, message) - } else { - !evalExpression(additionalOptions.customCondition, buildId, variables, message) - } + return !evalExpressionAsCode(additionalOptions.customCondition, buildId, variables, message) } return false @@ -260,8 +253,7 @@ object ControlUtils { buildId: String, runCondition: JobRunCondition, customCondition: String? = null, - message: StringBuilder = StringBuilder(), - asCodeEnabled: Boolean + message: StringBuilder = StringBuilder() ): Boolean { message.append( I18nUtil.getCodeLanMessage(BK_CHECK_JOB_RUN_CONDITION) @@ -278,11 +270,7 @@ object ControlUtils { false } // 条件全匹配就运行 JobRunCondition.CUSTOM_CONDITION_MATCH -> { // 满足以下自定义条件时运行 - return if (asCodeEnabled) { - !evalExpressionAsCode(customCondition, buildId, variables, message) - } else { - !evalExpression(customCondition, buildId, variables, message) - } + return !evalExpressionAsCode(customCondition, buildId, variables, message) } else -> { message.append(runCondition) @@ -311,18 +299,13 @@ object ControlUtils { buildId: String, runCondition: StageRunCondition, customCondition: String? = null, - message: StringBuilder = StringBuilder(), - asCodeEnabled: Boolean + message: StringBuilder = StringBuilder() ): Boolean { var skip = when (runCondition) { StageRunCondition.CUSTOM_VARIABLE_MATCH_NOT_RUN -> true // 条件匹配就跳过 StageRunCondition.CUSTOM_VARIABLE_MATCH -> false // 条件全匹配就运行 StageRunCondition.CUSTOM_CONDITION_MATCH -> { // 满足以下自定义条件时运行 - return if (asCodeEnabled) { - !evalExpressionAsCode(customCondition, buildId, variables, message) - } else { - !evalExpression(customCondition, buildId, variables, message) - } + return !evalExpressionAsCode(customCondition, buildId, variables, message) } else -> return false // 其它类型直接返回不跳过 } @@ -339,48 +322,6 @@ object ControlUtils { return skip } - private fun evalExpression( - customCondition: String?, - buildId: String, - variables: Map, - message: StringBuilder - ): Boolean { - return if (!customCondition.isNullOrBlank()) { - try { - val expressionResult = EvalExpress.eval(buildId, customCondition, variables) - logger.info( - "[$buildId]|STAGE_CONDITION|skip|CUSTOM_CONDITION_MATCH|expression=$customCondition" + - "|result=$expressionResult" - ) - message.append( - "Custom condition($customCondition) result is $expressionResult. " + - if (!expressionResult) { - " will be skipped! " - } else { - "" - } - ) - expressionResult - } catch (ignore: Exception) { - // 异常，则任务表达式为false - logger.info( - "[$buildId]|STAGE_CONDITION|skip|CUSTOM_CONDITION_MATCH|expression=$customCondition" + - "|result=exception: ${ignore.message}", - ignore - ) - message.append( - "Custom condition($customCondition) parse failed, will be skipped! Detail: ${ignore.message}" - ) - return false - } - } else { - // 空表达式也认为是false - logger.info("[$buildId]|STAGE_CONDITION|skip|CUSTOM_CONDITION_MATCH|expression is empty!") - message.append("Custom condition is empty, will be skipped!") - false - } - } - private fun evalExpressionAsCode( customCondition: String?, buildId: String, @@ -389,7 +330,13 @@ object ControlUtils { ): Boolean { return if (!customCondition.isNullOrBlank()) { try { - val expressionResult = ExpressionParser.evaluateByMap(customCondition, variables, false) + // 新增的表达式调用需要去掉兼容老流水线变量 + val variablesWithOutOld = variables.filter { PipelineVarUtil.oldVarToNewVar(it.key) == null } + val expressionResult = ExpressionParser.evaluateByMap( + expression = EnvReplacementParser.parse(customCondition, variables), + contextMap = variablesWithOutOld, + fetchValue = false + ) logger.info( "[$buildId]|EXPRESSION_CONDITION|skip|CUSTOM_CONDITION_MATCH|expression=$customCondition" + "|result=$expressionResult" @@ -400,31 +347,44 @@ object ControlUtils { expressionResult.toString().toBoolean() } message.append( - "Custom condition($customCondition) result is $expressionResult. " + - if (!resultIsTrue) { - " will be skipped! " - } else { - "" - } + I18nUtil.getCodeLanMessage( + messageCode = ProcessMessageCode.BK_PIPELINE_RUN_CONDITION_RESULT, + language = I18nUtil.getDefaultLocaleLanguage(), + params = arrayOf(customCondition, resultIsTrue.toString()) + ) + if (!resultIsTrue) { + I18nUtil.getCodeLanMessage( + messageCode = ProcessMessageCode.BK_PIPELINE_RUN_CONDITION_NOT_MATCH, + language = I18nUtil.getDefaultLocaleLanguage() + ) + } else { + "" + } ) resultIsTrue - } catch (ignore: ExpressionParseException) { + } catch (ignore: Throwable) { // 异常，则任务表达式为false - logger.info( + logger.warn( "[$buildId]|EXPRESSION_CONDITION|skip|CUSTOM_CONDITION_MATCH|expression=$customCondition" + "|result=exception: ${ignore.message}", ignore ) message.append( - "Custom condition($customCondition) parse failed, will be skipped! Detail: ${ignore.message}" + I18nUtil.getCodeLanMessage( + messageCode = ProcessMessageCode.BK_PIPELINE_RUN_CONDITION_WITH_ERROR, + language = I18nUtil.getDefaultLocaleLanguage(), + params = arrayOf(ignore.message ?: "") + ) + I18nUtil.getCodeLanMessage( + messageCode = ProcessMessageCode.BK_PIPELINE_RUN_CONDITION_NOT_MATCH, + language = I18nUtil.getDefaultLocaleLanguage() + ) ) - throw ignore + false } } else { // 空表达式也认为是false logger.info("[$buildId]|EXPRESSION_CONDITION|skip|CUSTOM_CONDITION_MATCH|expression is empty!") message.append("Custom condition is empty, will be skipped!") - throw ExpressionParseException(ParseExceptionKind.UnexpectedSymbol, null, "Custom condition is empty") + false } } diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/control/VmOperateTaskGenerator.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/control/VmOperateTaskGenerator.kt index 13fb819a3e0..e2575cf5a85 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/control/VmOperateTaskGenerator.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/control/VmOperateTaskGenerator.kt @@ -37,8 +37,8 @@ import com.tencent.devops.common.pipeline.pojo.element.RunCondition import com.tencent.devops.common.pipeline.type.BuildType import com.tencent.devops.process.engine.common.VMUtils import com.tencent.devops.process.engine.pojo.PipelineBuildTask -import org.springframework.stereotype.Component import javax.xml.bind.Element +import org.springframework.stereotype.Component /** * 生成运行环境操作的插件任务 @@ -55,11 +55,19 @@ class VmOperateTaskGenerator { fun isVmAtom(task: PipelineBuildTask) = isStartVM(task) || isStopVM(task) + fun isVmAtom(atomCode: String) = isStartVM(atomCode) || isStopVM(atomCode) + fun isStartVM(task: PipelineBuildTask) = task.taskAtom == START_VM_TASK_ATOM || task.taskAtom == START_NORMAL_TASK_ATOM fun isStopVM(task: PipelineBuildTask) = task.taskAtom == SHUTDOWN_VM_TASK_ATOM || task.taskAtom == SHUTDOWN_NORMAL_TASK_ATOM + + fun isStartVM(atomCode: String) = + atomCode.startsWith(START_VM_TASK_ATOM) || atomCode.startsWith(START_NORMAL_TASK_ATOM) + + fun isStopVM(atomCode: String) = + atomCode.startsWith(SHUTDOWN_VM_TASK_ATOM) || atomCode.startsWith(SHUTDOWN_NORMAL_TASK_ATOM) } /** @@ -120,7 +128,8 @@ class VmOperateTaskGenerator { subBuildId = null, additionalOptions = additionalOptions, atomCode = atomCode, - stepId = null + stepId = null, + jobId = container.jobId ) } @@ -182,7 +191,8 @@ class VmOperateTaskGenerator { subBuildId = null, additionalOptions = additionalOptions, atomCode = "$SHUTDOWN_VM_TASK_ATOM-END", - stepId = null + stepId = null, + jobId = container.jobId ) ) @@ -216,7 +226,8 @@ class VmOperateTaskGenerator { subBuildId = null, additionalOptions = additionalOptions, atomCode = "$SHUTDOWN_VM_TASK_ATOM-FINISH", - stepId = null + stepId = null, + jobId = container.jobId ) ) diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/control/lock/PipelineReleaseLock.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/control/lock/PipelineReleaseLock.kt new file mode 100644 index 00000000000..61f2da1d8e7 --- /dev/null +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/control/lock/PipelineReleaseLock.kt @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.process.engine.control.lock + +import com.tencent.devops.common.redis.RedisLock +import com.tencent.devops.common.redis.RedisOperation + +/** + * 流水线发布锁 + */ +class PipelineReleaseLock(redisOperation: RedisOperation, pipelineId: String) : + RedisLock( + redisOperation = redisOperation, + lockKey = "lock:pipeline:release:$pipelineId", + expiredTimeInSeconds = 30L, + sleepTime = 10L + ) { + override fun decorateKey(key: String): String { + // pipelineId在各集群唯一，key无需加上集群信息前缀来区分 + return key + } +} diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildContainerDao.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildContainerDao.kt index 6376b3e7e94..1b8ad9fe54d 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildContainerDao.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildContainerDao.kt @@ -63,6 +63,7 @@ class PipelineBuildContainerDao { SEQ, CONTAINER_ID, CONTAINER_HASH_ID, + JOB_ID, STATUS, START_TIME, END_TIME, @@ -81,6 +82,7 @@ class PipelineBuildContainerDao { buildContainer.seq, buildContainer.containerId, buildContainer.containerHashId, + buildContainer.jobId, buildContainer.status.ordinal, buildContainer.startTime, buildContainer.endTime, @@ -103,6 +105,7 @@ class PipelineBuildContainerDao { STAGE_ID, CONTAINER_ID, CONTAINER_HASH_ID, + JOB_ID, MATRIX_GROUP_FLAG, MATRIX_GROUP_ID, CONTAINER_TYPE, @@ -122,6 +125,7 @@ class PipelineBuildContainerDao { it.stageId, it.containerId, it.containerHashId, + it.jobId, it.matrixGroupFlag, it.matrixGroupId, it.containerType, @@ -153,6 +157,7 @@ class PipelineBuildContainerDao { .set(CONTAINER_TYPE, it.containerType) .set(CONTAINER_ID, it.containerId) .set(CONTAINER_HASH_ID, it.containerHashId) + .set(JOB_ID, it.jobId) .set(STATUS, it.status.ordinal) .set(START_TIME, it.startTime) .set(END_TIME, it.endTime) @@ -346,6 +351,7 @@ class PipelineBuildContainerDao { containerType = containerType, containerId = containerId, containerHashId = containerHashId, + jobId = jobId, matrixGroupFlag = matrixGroupFlag, matrixGroupId = matrixGroupId, seq = seq, diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildDao.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildDao.kt index 8cb0619eae9..8c6e707546e 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildDao.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildDao.kt @@ -49,6 +49,7 @@ import com.tencent.devops.model.process.tables.records.TPipelineBuildHistoryReco import com.tencent.devops.process.constant.ProcessMessageCode import com.tencent.devops.process.engine.pojo.BuildInfo import com.tencent.devops.process.engine.pojo.BuildRetryInfo +import com.tencent.devops.process.enums.HistorySearchType import com.tencent.devops.process.pojo.BuildStageStatus import com.tencent.devops.process.pojo.PipelineBuildMaterial import com.tencent.devops.process.pojo.app.StartBuildContext @@ -72,7 +73,7 @@ class PipelineBuildDao { companion object { private val mapper = PipelineBuildInfoJooqMapper() private val debugMapper = PipelineDebugBuildInfoJooqMapper() - private const val DEFAULT_PAGE_SIZE = 10 + private const val DEFAULT_PAGE_SIZE = 50 } fun create(dslContext: DSLContext, startBuildContext: StartBuildContext) { @@ -103,7 +104,8 @@ class PipelineBuildDao { BUILD_NUM_ALIAS, CONCURRENCY_GROUP, VERSION_NAME, - YAML_VERSION + YAML_VERSION, + EXECUTE_COUNT ).values( startBuildContext.buildId, startBuildContext.buildNum, @@ -127,7 +129,8 @@ class PipelineBuildDao { startBuildContext.buildNumAlias, startBuildContext.concurrencyGroup, startBuildContext.versionName, - startBuildContext.yamlVersion + startBuildContext.yamlVersion, + startBuildContext.executeCount ).execute() } } else { @@ -156,7 +159,8 @@ class PipelineBuildDao { BUILD_NUM_ALIAS, CONCURRENCY_GROUP, YAML_VERSION, - RESOURCE_MODEL + RESOURCE_MODEL, + EXECUTE_COUNT ).values( startBuildContext.buildId, startBuildContext.buildNum, @@ -180,7 +184,8 @@ class PipelineBuildDao { startBuildContext.buildNumAlias, startBuildContext.concurrencyGroup, startBuildContext.yamlVersion, - startBuildContext.debugModel?.let { self -> JsonUtil.toJson(self, formatted = false) } + startBuildContext.debugModelStr, + startBuildContext.executeCount ).execute() } } @@ -371,6 +376,8 @@ class PipelineBuildDao { with(T_PIPELINE_BUILD_HISTORY_DEBUG) { val conditions = mutableListOf() conditions.add(BUILD_ID.`in`(buildIds)) + // 增加过滤，对前端屏蔽已删除的构建 + conditions.add(DELETE_TIME.isNull) if (projectId != null) { conditions.add(PROJECT_ID.eq(projectId)) } @@ -412,6 +419,8 @@ class PipelineBuildDao { val select = dslContext.selectFrom(this) .where(PROJECT_ID.eq(projectId)) .and(PIPELINE_ID.eq(pipelineId)) + // 增加过滤，对前端屏蔽已删除的构建 + .and(DELETE_TIME.isNull) when (updateTimeDesc) { true -> select.orderBy(UPDATE_TIME.desc(), BUILD_ID) false -> select.orderBy(UPDATE_TIME.asc(), BUILD_ID) @@ -446,6 +455,8 @@ class PipelineBuildDao { .where(PROJECT_ID.eq(projectId)) .and(PIPELINE_ID.eq(pipelineId)) .and(VERSION.eq(debugVersion)) + // 增加过滤，对前端屏蔽已删除的构建 + .and(DELETE_TIME.isNull) .orderBy(BUILD_NUM.desc()) .limit(offset, limit) .fetch(0, Int::class.java) @@ -458,50 +469,55 @@ class PipelineBuildDao { projectId: String, pipelineId: String, buildNum: Int?, - statusSet: Set? + statusSet: Set?, + debug: Boolean ): BuildInfo? { - return with(T_PIPELINE_BUILD_HISTORY) { - val select = dslContext.selectFrom(this) - .where(PROJECT_ID.eq(projectId)) - .and(PIPELINE_ID.eq(pipelineId)) - - if (!statusSet.isNullOrEmpty()) { - select.and(STATUS.`in`(statusSet.map { it.ordinal })) - } + return if (debug) { + with(T_PIPELINE_BUILD_HISTORY_DEBUG) { + val select = dslContext.selectFrom(this) + .where(PROJECT_ID.eq(projectId)) + .and(PIPELINE_ID.eq(pipelineId)) + if (!statusSet.isNullOrEmpty()) { + select.and(STATUS.`in`(statusSet.map { it.ordinal })) + } - if (buildNum != null && buildNum > 0) { - select.and(BUILD_NUM.eq(buildNum)) - } else { // 取最新的 - select.orderBy(BUILD_NUM.desc()).limit(1) + if (buildNum != null && buildNum > 0) { + select.and(BUILD_NUM.eq(buildNum)) + } else { // 取最新的 + select.orderBy(BUILD_NUM.desc()).limit(1) + } + select.fetchOne(debugMapper) } - select.fetchOne(mapper) - } ?: with(T_PIPELINE_BUILD_HISTORY_DEBUG) { - val select = dslContext.selectFrom(this) - .where(PROJECT_ID.eq(projectId)) - .and(PIPELINE_ID.eq(pipelineId)) + } else { + with(T_PIPELINE_BUILD_HISTORY) { + val select = dslContext.selectFrom(this) + .where(PROJECT_ID.eq(projectId)) + .and(PIPELINE_ID.eq(pipelineId)) - if (!statusSet.isNullOrEmpty()) { - select.and(STATUS.`in`(statusSet.map { it.ordinal })) - } + if (!statusSet.isNullOrEmpty()) { + select.and(STATUS.`in`(statusSet.map { it.ordinal })) + } - if (buildNum != null && buildNum > 0) { - select.and(BUILD_NUM.eq(buildNum)) - } else { // 取最新的 - select.orderBy(BUILD_NUM.desc()).limit(1) + if (buildNum != null && buildNum > 0) { + select.and(BUILD_NUM.eq(buildNum)) + } else { // 取最新的 + select.orderBy(BUILD_NUM.desc()).limit(1) + } + select.fetchOne(mapper) } - select.fetchOne(debugMapper) } } fun getOneQueueBuild(dslContext: DSLContext, projectId: String, pipelineId: String): BuildInfo? { - return with(T_PIPELINE_BUILD_HISTORY) { + val release = with(T_PIPELINE_BUILD_HISTORY) { val select = dslContext.selectFrom(this) .where(PROJECT_ID.eq(projectId)) .and(PIPELINE_ID.eq(pipelineId)) .and(STATUS.`in`(setOf(BuildStatus.QUEUE.ordinal, BuildStatus.QUEUE_CACHE.ordinal))) .orderBy(BUILD_NUM.asc()).limit(1) select.fetchAny(mapper) - } ?: with(T_PIPELINE_BUILD_HISTORY_DEBUG) { + } + val debug = with(T_PIPELINE_BUILD_HISTORY_DEBUG) { val select = dslContext.selectFrom(this) .where(PROJECT_ID.eq(projectId)) .and(PIPELINE_ID.eq(pipelineId)) @@ -509,6 +525,12 @@ class PipelineBuildDao { .orderBy(BUILD_NUM.asc()).limit(1) select.fetchAny(debugMapper) } + return when { + release == null -> debug + debug == null -> release + release.queueTime > debug.queueTime -> debug + else -> release + } } fun getOneConcurrencyQueueBuild( @@ -517,7 +539,7 @@ class PipelineBuildDao { concurrencyGroup: String, pipelineId: String? = null ): BuildInfo? { - return with(T_PIPELINE_BUILD_HISTORY) { + val release = with(T_PIPELINE_BUILD_HISTORY) { val select = dslContext.selectFrom(this) .where(PROJECT_ID.eq(projectId)) .and(CONCURRENCY_GROUP.eq(concurrencyGroup)) @@ -527,7 +549,8 @@ class PipelineBuildDao { } select.orderBy(QUEUE_TIME.asc(), PIPELINE_ID, BUILD_NUM.asc()).limit(1) select.fetchAny(mapper) - } ?: with(T_PIPELINE_BUILD_HISTORY_DEBUG) { + } + val debug = with(T_PIPELINE_BUILD_HISTORY_DEBUG) { val select = dslContext.selectFrom(this) .where(PROJECT_ID.eq(projectId)) .and(CONCURRENCY_GROUP.eq(concurrencyGroup)) @@ -538,6 +561,12 @@ class PipelineBuildDao { select.orderBy(QUEUE_TIME.asc(), PIPELINE_ID, BUILD_NUM.asc()).limit(1) select.fetchAny(debugMapper) } + return when { + release == null -> debug + debug == null -> release + release.queueTime > debug.queueTime -> debug + else -> release + } } /** @@ -644,7 +673,9 @@ class PipelineBuildDao { val select = dslContext.selectFrom(this) .where( PIPELINE_ID.eq(pipelineId), - PROJECT_ID.eq(projectId) + PROJECT_ID.eq(projectId), + // 增加过滤，对前端屏蔽已删除的构建 + DELETE_TIME.isNull ) .orderBy(BUILD_NUM.desc()).limit(1) select.fetchAny(debugMapper) @@ -691,7 +722,9 @@ class PipelineBuildDao { BuildStatus.RUNNING.ordinal, // 3 运行中 BuildStatus.QUEUE.ordinal // 13 排队（新） ) - ) + ), + // 增加过滤，对前端屏蔽已删除的构建 + DELETE_TIME.isNull ) .orderBy(BUILD_NUM.desc()).limit(1) select.fetchAny(debugMapper) @@ -720,7 +753,9 @@ class PipelineBuildDao { .where( PIPELINE_ID.eq(pipelineId), PROJECT_ID.eq(projectId), - STATUS.eq(1) + STATUS.eq(1), + // 增加过滤，对前端屏蔽已删除的构建 + DELETE_TIME.isNull ) .orderBy(BUILD_NUM.desc()).limit(1) select.fetchAny(debugMapper) @@ -749,7 +784,9 @@ class PipelineBuildDao { .where( PIPELINE_ID.eq(pipelineId), PROJECT_ID.eq(projectId), - STATUS.eq(0) + STATUS.eq(0), + // 增加过滤，对前端屏蔽已删除的构建 + DELETE_TIME.isNull ) .orderBy(BUILD_NUM.desc()).limit(1) select.fetchAny(debugMapper) @@ -850,6 +887,8 @@ class PipelineBuildDao { if (startTimeEndTime != null && startTimeEndTime > 0) { where.and(START_TIME.le(Timestamp(startTimeEndTime).toLocalDateTime())) } + // 增加过滤，对前端屏蔽已删除的构建 + where.and(DELETE_TIME.isNull) where.fetchOne(0, Int::class.java)!! } } @@ -887,6 +926,8 @@ class PipelineBuildDao { if (startTimeEndTime != null && startTimeEndTime > 0) { where.and(START_TIME.le(Timestamp(startTimeEndTime).toLocalDateTime())) } + // 增加过滤，对前端屏蔽已删除的构建 + where.and(DELETE_TIME.isNull) where.fetchOne(0, Int::class.java)!! } } @@ -916,7 +957,9 @@ class PipelineBuildDao { buildNoEnd: Int?, buildMsg: String?, startUser: List?, - debugVersion: Int? + debugVersion: Int?, + triggerAlias: List?, + triggerBranch: List? ): Int { return if (debugVersion == null) { with(T_PIPELINE_BUILD_HISTORY) { @@ -943,7 +986,9 @@ class PipelineBuildDao { remark = remark, buildNoStart = buildNoStart, buildNoEnd = buildNoEnd, - buildMsg = buildMsg + buildMsg = buildMsg, + triggerAlias = triggerAlias, + triggerBranch = triggerBranch ) where.fetchOne(0, Int::class.java)!! } @@ -974,7 +1019,9 @@ class PipelineBuildDao { remark = remark, buildNoStart = buildNoStart, buildNoEnd = buildNoEnd, - buildMsg = buildMsg + buildMsg = buildMsg, + triggerAlias = triggerAlias, + triggerBranch = triggerBranch ) where.fetchOne(0, Int::class.java)!! } @@ -1008,7 +1055,9 @@ class PipelineBuildDao { buildMsg: String?, startUser: List?, updateTimeDesc: Boolean? = null, - debugVersion: Int? + debugVersion: Int?, + triggerAlias: List?, + triggerBranch: List? ): Collection { return if (debugVersion == null) { with(T_PIPELINE_BUILD_HISTORY) { @@ -1034,7 +1083,9 @@ class PipelineBuildDao { remark = remark, buildNoStart = buildNoStart, buildNoEnd = buildNoEnd, - buildMsg = buildMsg + buildMsg = buildMsg, + triggerAlias = triggerAlias, + triggerBranch = triggerBranch ) when (updateTimeDesc) { @@ -1071,10 +1122,10 @@ class PipelineBuildDao { remark = remark, buildNoStart = buildNoStart, buildNoEnd = buildNoEnd, - buildMsg = buildMsg + buildMsg = buildMsg, + triggerAlias = triggerAlias, + triggerBranch = triggerBranch ) - // 增加过滤，对前端屏蔽已删除的构建 - where.and(DELETE_TIME.isNull) when (updateTimeDesc) { true -> where.orderBy(UPDATE_TIME.desc(), BUILD_ID) false -> where.orderBy(UPDATE_TIME.asc(), BUILD_ID) @@ -1107,7 +1158,9 @@ class PipelineBuildDao { remark: String?, buildNoStart: Int?, buildNoEnd: Int?, - buildMsg: String? + buildMsg: String?, + triggerAlias: List?, + triggerBranch: List? ) { if (!materialAlias.isNullOrEmpty() && materialAlias.first().isNotBlank()) { var conditionsOr: Condition @@ -1205,6 +1258,36 @@ class PipelineBuildDao { if (!buildMsg.isNullOrBlank()) { where.and(BUILD_MSG.like("%$buildMsg%")) } + if (!triggerAlias.isNullOrEmpty() && triggerAlias.first().isNotBlank()) { + var conditionsOr: Condition + + conditionsOr = JooqUtils.jsonExtract(t1 = WEBHOOK_INFO, t2 = "\$.webhookAliasName", lower = true) + .like("%${triggerAlias.first().lowercase()}%") + + triggerAlias.forEachIndexed { index, s -> + if (index == 0) return@forEachIndexed + conditionsOr = conditionsOr.or( + JooqUtils.jsonExtract(WEBHOOK_INFO, "\$.webhookAliasName", lower = true) + .like("%${s.lowercase()}%") + ) + } + where.and(conditionsOr) + } + if (!triggerBranch.isNullOrEmpty() && triggerBranch.first().isNotBlank()) { + var conditionsOr: Condition + + conditionsOr = JooqUtils.jsonExtract(WEBHOOK_INFO, "\$.webhookBranch", lower = true) + .like("%${triggerBranch.first().lowercase()}%") + + triggerBranch.forEachIndexed { index, s -> + if (index == 0) return@forEachIndexed + conditionsOr = conditionsOr.or( + JooqUtils.jsonExtract(WEBHOOK_INFO, "\$.webhookBranch", lower = true) + .like("%${s.lowercase()}%") + ) + } + where.and(conditionsOr) + } } private fun TPipelineBuildHistoryDebug.makeDebugCondition( @@ -1228,8 +1311,12 @@ class PipelineBuildDao { remark: String?, buildNoStart: Int?, buildNoEnd: Int?, - buildMsg: String? + buildMsg: String?, + triggerAlias: List?, + triggerBranch: List? ) { + // 增加过滤，对前端屏蔽已删除的构建 + where.and(DELETE_TIME.isNull) if (!materialAlias.isNullOrEmpty() && materialAlias.first().isNotBlank()) { var conditionsOr: Condition @@ -1326,6 +1413,36 @@ class PipelineBuildDao { if (!buildMsg.isNullOrBlank()) { where.and(BUILD_MSG.like("%$buildMsg%")) } + if (!triggerAlias.isNullOrEmpty() && triggerAlias.first().isNotBlank()) { + var conditionsOr: Condition + + conditionsOr = JooqUtils.jsonExtract(t1 = WEBHOOK_INFO, t2 = "\$.webhookAliasName", lower = true) + .like("%${triggerAlias.first().lowercase()}%") + + triggerAlias.forEachIndexed { index, s -> + if (index == 0) return@forEachIndexed + conditionsOr = conditionsOr.or( + JooqUtils.jsonExtract(WEBHOOK_INFO, "\$.webhookAliasName", lower = true) + .like("%${s.lowercase()}%") + ) + } + where.and(conditionsOr) + } + if (!triggerBranch.isNullOrEmpty() && triggerBranch.first().isNotBlank()) { + var conditionsOr: Condition + + conditionsOr = JooqUtils.jsonExtract(WEBHOOK_INFO, "\$.webhookBranch", lower = true) + .like("%${triggerBranch.first().lowercase()}%") + + triggerBranch.forEachIndexed { index, s -> + if (index == 0) return@forEachIndexed + conditionsOr = conditionsOr.or( + JooqUtils.jsonExtract(WEBHOOK_INFO, "\$.webhookBranch", lower = true) + .like("%${s.lowercase()}%") + ) + } + where.and(conditionsOr) + } } fun updateBuildRemark( @@ -1370,25 +1487,43 @@ class PipelineBuildDao { } } - fun getBuildHistoryMaterial( + /** + * 构建历史搜索下拉框 + */ + fun listHistorySearchOptions( dslContext: DSLContext, projectId: String, pipelineId: String, - debugVersion: Int? + debugVersion: Int?, + type: HistorySearchType ): Collection { return if (debugVersion == null) { with(T_PIPELINE_BUILD_HISTORY) { - dslContext.selectFrom(this) + val where = dslContext.selectFrom(this) .where(PIPELINE_ID.eq(pipelineId).and(PROJECT_ID.eq(projectId))) - .orderBy(BUILD_NUM.desc()).limit(DEFAULT_PAGE_SIZE) + when (type) { + HistorySearchType.MATERIAL -> + where.and(MATERIAL.isNotNull) + + HistorySearchType.TRIGGER -> + where.and(WEBHOOK_INFO.isNotNull) + } + where.orderBy(BUILD_NUM.desc()).limit(DEFAULT_PAGE_SIZE) .fetch(mapper) } } else { with(T_PIPELINE_BUILD_HISTORY_DEBUG) { - dslContext.selectFrom(this) + val where = dslContext.selectFrom(this) .where(PIPELINE_ID.eq(pipelineId).and(PROJECT_ID.eq(projectId))) .and(VERSION.eq(debugVersion)) - .orderBy(BUILD_NUM.desc()).limit(DEFAULT_PAGE_SIZE) + when (type) { + HistorySearchType.MATERIAL -> + where.and(MATERIAL.isNotNull) + + HistorySearchType.TRIGGER -> + where.and(WEBHOOK_INFO.isNotNull) + } + where.orderBy(BUILD_NUM.desc()).limit(DEFAULT_PAGE_SIZE) .fetch(debugMapper) } } @@ -1523,7 +1658,7 @@ class PipelineBuildDao { .execute() == 1 } } - return if (!success) with(T_PIPELINE_BUILD_HISTORY) { + return if (!success) with(T_PIPELINE_BUILD_HISTORY_DEBUG) { val update = dslContext.update(this) .set(STAGE_STATUS, JsonUtil.toJson(stageStatus, formatted = false)) newBuildStatus?.let { update.set(STATUS, it.ordinal) } @@ -1697,14 +1832,14 @@ class PipelineBuildDao { dslContext: DSLContext, projectId: String, pipelineId: String, - version: Int + version: Int? = null ): List { with(T_PIPELINE_BUILD_HISTORY_DEBUG) { - return dslContext.selectFrom(this) + val select = dslContext.selectFrom(this) .where(PROJECT_ID.eq(projectId).and(PIPELINE_ID.eq(pipelineId))) - .and(VERSION.eq(version)) .and(DELETE_TIME.isNotNull) - .fetch(debugMapper) + version?.let { select.and(VERSION.eq(version)) } + return select.fetch(debugMapper) } } @@ -1712,15 +1847,15 @@ class PipelineBuildDao { dslContext: DSLContext, projectId: String, pipelineId: String, - version: Int + version: Int? = null ): Int { with(T_PIPELINE_BUILD_HISTORY_DEBUG) { val now = LocalDateTime.now() - return dslContext.update(this) + val update = dslContext.update(this) .set(DELETE_TIME, now) .where(PROJECT_ID.eq(projectId).and(PIPELINE_ID.eq(pipelineId))) - .and(VERSION.eq(version)) - .execute() + version?.let { update.and(VERSION.eq(version)) } + return update.execute() } } diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildStageDao.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildStageDao.kt index 18c58fad31d..cdbbc0134d7 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildStageDao.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildStageDao.kt @@ -37,6 +37,7 @@ import com.tencent.devops.model.process.tables.records.TPipelineBuildStageRecord import com.tencent.devops.process.engine.common.Timeout import com.tencent.devops.process.engine.pojo.PipelineBuildStage import com.tencent.devops.process.engine.pojo.PipelineBuildStageControlOption +import org.jooq.Condition import org.jooq.DSLContext import org.jooq.DatePart import org.jooq.RecordMapper @@ -198,11 +199,25 @@ class PipelineBuildStageDao { } } - fun listBuildStages(dslContext: DSLContext, projectId: String, buildId: String): List { + fun listBuildStages( + dslContext: DSLContext, + projectId: String, + buildId: String, + statusSet: Set? = null, + num: Int? = null + ): List { return with(T_PIPELINE_BUILD_STAGE) { - dslContext.selectFrom(this) - .where(BUILD_ID.eq(buildId).and(PROJECT_ID.eq(projectId))) - .orderBy(SEQ.asc()).fetch(mapper) + val conditions = mutableListOf() + conditions.add(BUILD_ID.eq(buildId)) + conditions.add(PROJECT_ID.eq(projectId)) + if (!statusSet.isNullOrEmpty()) { + conditions.add(STATUS.`in`(statusSet.map { it.ordinal })) + } + val baseStep = dslContext.selectFrom(this).where(conditions).orderBy(SEQ.asc()) + if (num != null) { + baseStep.limit(num) + } + baseStep.fetch(mapper) } } @@ -263,12 +278,13 @@ class PipelineBuildStageDao { buildId: String, statusSet: Set ): PipelineBuildStage? { - with(T_PIPELINE_BUILD_STAGE) { - return dslContext.selectFrom(this) - .where(PROJECT_ID.eq(projectId)).and(BUILD_ID.eq(buildId)) - .and(STATUS.`in`(statusSet.map { it.ordinal })) - .orderBy(SEQ.asc()).limit(1).fetchOne(mapper) - } + return listBuildStages( + dslContext = dslContext, + projectId = projectId, + buildId = buildId, + statusSet = statusSet, + num = 1 + ).getOrNull(0) } class PipelineBuildStageJooqMapper : RecordMapper { diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildSummaryDao.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildSummaryDao.kt index 43e5534500e..a4829972d63 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildSummaryDao.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildSummaryDao.kt @@ -158,23 +158,11 @@ class PipelineBuildSummaryDao { dslContext.update(this) .set(numColumn, numColumn + 1) .set(BUILD_NUM_ALIAS, buildNumAlias) - .let { - /* debug 模式构建不需要更新状态 */ - if (!debug) { - it.set(LATEST_BUILD_ID, buildId).set(LATEST_STATUS, BuildStatus.QUEUE.ordinal) - } else it - } .where(PIPELINE_ID.eq(pipelineId).and(PROJECT_ID.eq(projectId))).execute() } else { dslContext.update(this) .set(numColumn, buildNum) .set(BUILD_NUM_ALIAS, buildNumAlias) - .let { - /* debug 模式构建不需要更新状态 */ - if (!debug) { - it.set(LATEST_BUILD_ID, buildId).set(LATEST_STATUS, BuildStatus.QUEUE.ordinal) - } else it - } .where(PIPELINE_ID.eq(pipelineId).and(PROJECT_ID.eq(projectId))).execute() } return dslContext.select(numColumn) @@ -503,31 +491,44 @@ class PipelineBuildSummaryDao { fun startLatestRunningBuild( dslContext: DSLContext, latestRunningBuild: LatestRunningBuild, - executeCount: Int + executeCount: Int, + debug: Boolean ): Int { return with(latestRunningBuild) { - with(T_PIPELINE_BUILD_SUMMARY) { - dslContext.update(this) - .let { - if (executeCount == 1) { - // 只有首次才写入LATEST_BUILD_ID - it.set(LATEST_BUILD_ID, buildId).set(LATEST_STATUS, status.ordinal) - } else { - // 重试时只有最新的构建才能刷新LATEST_STATUS - it.set( - LATEST_STATUS, - DSL.`when`(LATEST_BUILD_ID.eq(buildId), status.ordinal).otherwise(LATEST_STATUS) - ) + if (debug) { + /* debug 下只更新计数 */ + with(T_PIPELINE_BUILD_SUMMARY) { + dslContext.update(this) + .set(QUEUE_COUNT, QUEUE_COUNT - 1) + .set(RUNNING_COUNT, RUNNING_COUNT + 1) + .where(PROJECT_ID.eq(projectId)) + .and(PIPELINE_ID.eq(pipelineId)) + .execute() + } + } else { + with(T_PIPELINE_BUILD_SUMMARY) { + dslContext.update(this) + .let { + if (executeCount == 1) { + // 只有首次才写入LATEST_BUILD_ID + it.set(LATEST_BUILD_ID, buildId).set(LATEST_STATUS, status.ordinal) + } else { + // 重试时只有最新的构建才能刷新LATEST_STATUS + it.set( + LATEST_STATUS, + DSL.`when`(LATEST_BUILD_ID.eq(buildId), status.ordinal).otherwise(LATEST_STATUS) + ) + } } - } - .set(LATEST_TASK_COUNT, taskCount) - .set(LATEST_START_USER, userId) - .set(QUEUE_COUNT, QUEUE_COUNT - 1) - .set(RUNNING_COUNT, RUNNING_COUNT + 1) - .set(LATEST_START_TIME, LocalDateTime.now()) - .where(PROJECT_ID.eq(projectId)) - .and(PIPELINE_ID.eq(pipelineId)) // 并发的情况下，不再考虑LATEST_BUILD_ID，没有意义，而且会造成Slow SQL - .execute() + .set(LATEST_TASK_COUNT, taskCount) + .set(LATEST_START_USER, userId) + .set(QUEUE_COUNT, QUEUE_COUNT - 1) + .set(RUNNING_COUNT, RUNNING_COUNT + 1) + .set(LATEST_START_TIME, LocalDateTime.now()) + .where(PROJECT_ID.eq(projectId)) + .and(PIPELINE_ID.eq(pipelineId)) // 并发的情况下，不再考虑LATEST_BUILD_ID，没有意义，而且会造成Slow SQL + .execute() + } } } } @@ -562,7 +563,8 @@ class PipelineBuildSummaryDao { fun finishLatestRunningBuild( dslContext: DSLContext, latestRunningBuild: LatestRunningBuild, - isStageFinish: Boolean + isStageFinish: Boolean, + debug: Boolean ) { return with(latestRunningBuild) { with(T_PIPELINE_BUILD_SUMMARY) { @@ -571,12 +573,14 @@ class PipelineBuildSummaryDao { .set( LATEST_STATUS, DSL.`when`(LATEST_BUILD_ID.eq(buildId), status.ordinal).otherwise(LATEST_STATUS) - ) // 不一定是FINISH，也有可能其它失败的status - .set(LATEST_END_TIME, endTime) // 结束时间 + ) + if (!debug) { + // 不一定是FINISH，也有可能其它失败的status + update.set(LATEST_END_TIME, endTime) // 结束时间 .set(LATEST_TASK_ID, "") // 结束时清空 .set(LATEST_TASK_NAME, "") // 结束时清空 .set(FINISH_COUNT, FINISH_COUNT + 1) - + } if (!isStageFinish) update.set(RUNNING_COUNT, RUNNING_COUNT - 1) update.where(PROJECT_ID.eq(projectId)) .and(PIPELINE_ID.eq(pipelineId)) // 并发的情况下，不用考虑是否是当前的LATEST_BUILD_ID，而且会造成Slow SQL @@ -593,21 +597,24 @@ class PipelineBuildSummaryDao { projectId: String, pipelineId: String, buildId: String, - runningIncrement: Int = 1 + runningIncrement: Int = 1, + debug: Boolean ) { with(T_PIPELINE_BUILD_SUMMARY) { val update = dslContext.update(this).set(RUNNING_COUNT, RUNNING_COUNT + runningIncrement) - - if (runningIncrement > 0) { - update.set( - LATEST_STATUS, - DSL.`when`(LATEST_BUILD_ID.eq(buildId), BuildStatus.RUNNING.ordinal).otherwise(LATEST_STATUS) - ) - } else { - update.set( - LATEST_STATUS, - DSL.`when`(LATEST_BUILD_ID.eq(buildId), BuildStatus.STAGE_SUCCESS.ordinal).otherwise(LATEST_STATUS) - ).set(LATEST_END_TIME, LocalDateTime.now()) + if (!debug) { + if (runningIncrement > 0) { + update.set( + LATEST_STATUS, + DSL.`when`(LATEST_BUILD_ID.eq(buildId), BuildStatus.RUNNING.ordinal).otherwise(LATEST_STATUS) + ) + } else { + update.set( + LATEST_STATUS, + DSL.`when`(LATEST_BUILD_ID.eq(buildId), BuildStatus.STAGE_SUCCESS.ordinal) + .otherwise(LATEST_STATUS) + ).set(LATEST_END_TIME, LocalDateTime.now()) + } } update.where(PROJECT_ID.eq(projectId)) .and(PIPELINE_ID.eq(pipelineId)) // 并发的情况下，不用考虑是否是当前的LATEST_BUILD_ID，而且会造成Slow SQL diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildTaskDao.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildTaskDao.kt index 5a1b3c922d0..c6e655df7d6 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildTaskDao.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildTaskDao.kt @@ -66,6 +66,7 @@ class PipelineBuildTaskDao { CONTAINER_TYPE, CONTAINER_ID, CONTAINER_HASH_ID, + JOB_ID, TASK_SEQ, TASK_ID, STEP_ID, @@ -90,6 +91,7 @@ class PipelineBuildTaskDao { buildTask.containerType, buildTask.containerId, buildTask.containerHashId, + buildTask.jobId, buildTask.taskSeq, buildTask.taskId, buildTask.stepId, @@ -143,6 +145,7 @@ class PipelineBuildTaskDao { ERROR_CODE, ERROR_MSG, CONTAINER_HASH_ID, + JOB_ID, ATOM_CODE ).also { insert -> taskList.forEach { @@ -178,6 +181,7 @@ class PipelineBuildTaskDao { it.errorCode, it.errorMsg?.coerceAtMaxLength(PIPELINE_TASK_MESSAGE_STRING_LENGTH_MAX), it.containerHashId, + it.jobId, it.atomCode ) } @@ -216,6 +220,7 @@ class PipelineBuildTaskDao { .set(ERROR_MSG, it.errorMsg?.coerceAtMaxLength(PIPELINE_TASK_MESSAGE_STRING_LENGTH_MAX)) .set(ERROR_CODE, it.errorCode) .set(CONTAINER_HASH_ID, it.containerHashId) + .set(JOB_ID, it.jobId) .set(ATOM_CODE, it.atomCode) .where(BUILD_ID.eq(it.buildId).and(TASK_ID.eq(it.taskId)).and(PROJECT_ID.eq(it.projectId))) .execute() @@ -402,6 +407,7 @@ class PipelineBuildTaskDao { containerId = containerId, containerHashId = containerHashId, containerType = containerType, + jobId = jobId, taskSeq = taskSeq, taskId = taskId, stepId = stepId, diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildVarDao.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildVarDao.kt index db547242c93..d674f74a442 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildVarDao.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineBuildVarDao.kt @@ -114,6 +114,36 @@ class PipelineBuildVarDao @Autowired constructor() { } } + fun getBuildVarMap( + dslContext: DSLContext, + projectId: String, + buildId: String, + keys: Set? = null + ): Map { + with(T_PIPELINE_BUILD_VAR) { + val where = dslContext.selectFrom(this) + .where(BUILD_ID.eq(buildId).and(PROJECT_ID.eq(projectId))) + if (!keys.isNullOrEmpty()) { + where.and(KEY.`in`(keys)) + } + return where.fetch().associateBy( + { it.key }, + { + if (it.varType != null) { + BuildParameters( + key = it.key, + value = it.value, + valueType = BuildFormPropertyType.valueOf(it.varType), + readOnly = it.readOnly + ) + } else { + BuildParameters(key = it.key, value = it.value, readOnly = it.readOnly) + } + } + ) + } + } + fun getVarsWithType( dslContext: DSLContext, projectId: String, @@ -132,9 +162,14 @@ class PipelineBuildVarDao @Autowired constructor() { val list = mutableListOf() result.forEach { if (it.varType != null) { - list.add(BuildParameters(it.key, it.value, BuildFormPropertyType.valueOf(it.varType))) + list.add(BuildParameters( + key = it.key, + value = it.value, + valueType = BuildFormPropertyType.valueOf(it.varType), + readOnly = it.readOnly + )) } else { - list.add(BuildParameters(it.key, it.value)) + list.add(BuildParameters(key = it.key, value = it.value, readOnly = it.readOnly)) } } return list diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineInfoDao.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineInfoDao.kt index c6a24b22e35..ec7e213bb72 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineInfoDao.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineInfoDao.kt @@ -121,7 +121,8 @@ class PipelineInfoDao { taskCount: Int = 0, latestVersion: Int = 0, updateLastModifyUser: Boolean? = true, - latestVersionStatus: VersionStatus? = null + latestVersionStatus: VersionStatus? = null, + locked: Boolean? = null ): Boolean { return with(T_PIPELINE_INFO) { val update = dslContext.update(this) @@ -143,6 +144,9 @@ class PipelineInfoDao { if (taskCount > 0) { update.set(TASK_COUNT, taskCount) } + if (locked != null) { + update.set(LOCKED, locked) + } val conditions = mutableListOf() conditions.add(PROJECT_ID.eq(projectId)) conditions.add(PIPELINE_ID.eq(pipelineId)) @@ -632,7 +636,8 @@ class PipelineInfoDao { id = id, latestVersionStatus = latestVersionStatus?.let { VersionStatus.valueOf(it) - } ?: VersionStatus.RELEASED + } ?: VersionStatus.RELEASED, + locked = t.locked ) } } else { diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineResourceDao.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineResourceDao.kt index 7bc449ce0f5..78f1d5247db 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineResourceDao.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineResourceDao.kt @@ -275,6 +275,7 @@ class PipelineResourceDao { yaml = record.yaml, yamlVersion = record.yamlVersion, creator = record.creator ?: "unknown", + updater = null, versionName = versionName, createTime = record.createTime, updateTime = null, diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineResourceVersionDao.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineResourceVersionDao.kt index dc5d01035c4..a80262393b2 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineResourceVersionDao.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/dao/PipelineResourceVersionDao.kt @@ -34,6 +34,7 @@ import com.tencent.devops.common.pipeline.enums.BranchVersionAction import com.tencent.devops.common.pipeline.enums.VersionStatus import com.tencent.devops.model.process.Tables.T_PIPELINE_RESOURCE_VERSION import com.tencent.devops.model.process.tables.records.TPipelineResourceVersionRecord +import com.tencent.devops.process.engine.pojo.PipelineInfo import com.tencent.devops.process.pojo.pipeline.PipelineResourceVersion import com.tencent.devops.process.pojo.setting.PipelineVersionSimple import com.tencent.devops.process.utils.PipelineVersionUtils @@ -55,9 +56,9 @@ class PipelineResourceVersionDao { fun create( dslContext: DSLContext, + userId: String, projectId: String, pipelineId: String, - creator: String, version: Int, versionName: String, model: Model, @@ -71,48 +72,9 @@ class PipelineResourceVersionDao { versionStatus: VersionStatus?, branchAction: BranchVersionAction?, description: String? - ): TPipelineResourceVersionRecord? { - return create( - dslContext = dslContext, - projectId = projectId, - pipelineId = pipelineId, - creator = creator, - version = version, - versionName = versionName, - modelStr = JsonUtil.toJson(model, formatted = false), - yamlStr = yamlStr, - yamlVersion = yamlVersion, - baseVersion = baseVersion, - versionNum = versionNum, - pipelineVersion = pipelineVersion, - triggerVersion = triggerVersion, - settingVersion = settingVersion, - versionStatus = versionStatus, - branchAction = branchAction, - description = description - ) - } - - fun create( - dslContext: DSLContext, - projectId: String, - pipelineId: String, - creator: String, - version: Int, - versionName: String, - modelStr: String, - baseVersion: Int?, - yamlStr: String?, - yamlVersion: String?, - versionNum: Int?, - pipelineVersion: Int?, - triggerVersion: Int?, - settingVersion: Int?, - versionStatus: VersionStatus?, - branchAction: BranchVersionAction?, - description: String? ): TPipelineResourceVersionRecord? { with(T_PIPELINE_RESOURCE_VERSION) { + val modelStr = JsonUtil.toJson(model, formatted = false) return dslContext.insertInto(this) .set(PROJECT_ID, projectId) .set(PIPELINE_ID, pipelineId) @@ -121,7 +83,8 @@ class PipelineResourceVersionDao { .set(MODEL, modelStr) .set(YAML, yamlStr) .set(YAML_VERSION, yamlVersion) - .set(CREATOR, creator) + .set(CREATOR, userId) + .set(UPDATER, userId) .set(CREATE_TIME, LocalDateTime.now()) .set(VERSION_NUM, versionNum) .set(PIPELINE_VERSION, pipelineVersion) @@ -136,7 +99,7 @@ class PipelineResourceVersionDao { .set(MODEL, modelStr) .set(YAML, yamlStr) .set(YAML_VERSION, yamlVersion) - .set(CREATOR, creator) + .set(UPDATER, userId) .set(VERSION_NUM, versionNum) .set(VERSION_NAME, versionName) .set(BASE_VERSION, baseVersion) @@ -205,19 +168,19 @@ class PipelineResourceVersionDao { dslContext: DSLContext, projectId: String, pipelineId: String, - branchName: String + branchName: String? ): PipelineResourceVersion? { // 一定是取最新的分支版本 with(T_PIPELINE_RESOURCE_VERSION) { - return dslContext.selectFrom(this) + val select = dslContext.selectFrom(this) .where(PIPELINE_ID.eq(pipelineId).and(PROJECT_ID.eq(projectId))) .and(STATUS.eq(VersionStatus.BRANCH.name)) .and( BRANCH_ACTION.ne(BranchVersionAction.INACTIVE.name) .or(BRANCH_ACTION.isNull) ) - .and(VERSION_NAME.eq(branchName)) - .orderBy(VERSION.desc()).limit(1) + branchName?.let { select.and(VERSION_NAME.eq(branchName)) } + return select.orderBy(VERSION.desc()).limit(1) .fetchAny(mapper) } } @@ -303,6 +266,7 @@ class PipelineResourceVersionDao { with(T_PIPELINE_RESOURCE_VERSION) { val update = dslContext.update(this) .set(BRANCH_ACTION, BranchVersionAction.INACTIVE.name) + .set(UPDATE_TIME, UPDATE_TIME) .where(PIPELINE_ID.eq(pipelineId).and(PROJECT_ID.eq(projectId))) .and(STATUS.eq(VersionStatus.BRANCH.name)) .and( @@ -334,6 +298,7 @@ class PipelineResourceVersionDao { return with(T_PIPELINE_RESOURCE_VERSION) { dslContext.update(this) .set(STATUS, VersionStatus.DELETE.name) + .set(UPDATE_TIME, UPDATE_TIME) .where(PIPELINE_ID.eq(pipelineId)) .and(VERSION.eq(version)) .and(PROJECT_ID.eq(projectId)) @@ -345,6 +310,7 @@ class PipelineResourceVersionDao { dslContext: DSLContext, projectId: String, pipelineId: String, + pipelineInfo: PipelineInfo, queryUnknownRelatedFlag: Boolean? = null, maxQueryVersion: Int? = null, offset: Int, @@ -391,7 +357,11 @@ class PipelineResourceVersionDao { maxQueryVersion?.let { query.and(VERSION.le(maxQueryVersion)) } - return query.orderBy(VERSION.desc()).limit(limit).offset(offset).fetch(sampleMapper) + val list = query.orderBy( + UPDATE_TIME.desc(), VERSION_NUM.desc(), VERSION.desc() + ).limit(limit).offset(offset).fetch(sampleMapper) + list.forEach { if (it.version == pipelineInfo.version) it.latestReleasedFlag = true } + return list } } @@ -498,6 +468,7 @@ class PipelineResourceVersionDao { with(T_PIPELINE_RESOURCE_VERSION) { return dslContext.update(this) .set(DEBUG_BUILD_ID, debugBuildId) + .set(UPDATE_TIME, UPDATE_TIME) .where(PIPELINE_ID.eq(pipelineId).and(PROJECT_ID.eq(projectId)).and(VERSION.eq(version))) .execute() == 1 } @@ -535,6 +506,7 @@ class PipelineResourceVersionDao { with(T_PIPELINE_RESOURCE_VERSION) { val baseStep = dslContext.update(this) .set(REFER_COUNT, referCount) + .set(UPDATE_TIME, UPDATE_TIME) referFlag?.let { baseStep.set(REFER_FLAG, referFlag) } baseStep.where(PIPELINE_ID.eq(pipelineId).and(PROJECT_ID.eq(projectId)).and(VERSION.`in`(versions))) .execute() @@ -543,6 +515,7 @@ class PipelineResourceVersionDao { fun updateSettingVersion( dslContext: DSLContext, + userId: String, projectId: String, pipelineId: String, version: Int, @@ -551,6 +524,7 @@ class PipelineResourceVersionDao { with(T_PIPELINE_RESOURCE_VERSION) { return dslContext.update(this) .set(SETTING_VERSION, settingVersion) + .set(UPDATER, userId) .where(PIPELINE_ID.eq(pipelineId).and(PROJECT_ID.eq(projectId)).and(VERSION.eq(version))) .execute() } @@ -558,6 +532,7 @@ class PipelineResourceVersionDao { fun updateBranchVersion( dslContext: DSLContext, + userId: String, projectId: String, pipelineId: String, branchName: String?, @@ -565,6 +540,7 @@ class PipelineResourceVersionDao { ): Int { with(T_PIPELINE_RESOURCE_VERSION) { val update = dslContext.update(this) + .set(UPDATER, userId) .set(BRANCH_ACTION, branchVersionAction.name) .where( PIPELINE_ID.eq(pipelineId) @@ -606,6 +582,7 @@ class PipelineResourceVersionDao { yaml = record.yaml, yamlVersion = record.yamlVersion, creator = record.creator, + updater = record.updater, versionName = versionName, createTime = record.createTime, updateTime = record.updateTime, @@ -642,6 +619,7 @@ class PipelineResourceVersionDao { pipelineId = record.pipelineId, creator = record.creator ?: "unknown", createTime = record.createTime.timestampmilli(), + updater = record.updater, updateTime = record.updateTime.timestampmilli(), version = record.version ?: 1, versionName = versionName, diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/extend/DefaultModelCheckPlugin.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/extend/DefaultModelCheckPlugin.kt index eace6a8a57c..f204a7e1a20 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/extend/DefaultModelCheckPlugin.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/extend/DefaultModelCheckPlugin.kt @@ -38,9 +38,11 @@ import com.tencent.devops.common.pipeline.container.Stage import com.tencent.devops.common.pipeline.container.TriggerContainer import com.tencent.devops.common.pipeline.container.VMBuildContainer import com.tencent.devops.common.pipeline.enums.JobRunCondition +import com.tencent.devops.common.pipeline.enums.StageRunCondition import com.tencent.devops.common.pipeline.extend.ModelCheckPlugin import com.tencent.devops.common.pipeline.pojo.BuildFormProperty import com.tencent.devops.common.pipeline.pojo.element.Element +import com.tencent.devops.common.pipeline.pojo.element.RunCondition import com.tencent.devops.common.pipeline.pojo.element.atom.BeforeDeleteParam import com.tencent.devops.common.pipeline.pojo.element.atom.ElementCheckResult import com.tencent.devops.common.pipeline.pojo.element.atom.PipelineCheckFailedErrors @@ -65,6 +67,7 @@ import com.tencent.devops.process.utils.DependOnUtils import com.tencent.devops.process.utils.KEY_JOB import com.tencent.devops.process.utils.KEY_STAGE import com.tencent.devops.process.utils.KEY_TASK +import com.tencent.devops.process.utils.PIPELINE_CONDITION_EXPRESSION_LENGTH_MAX import com.tencent.devops.process.utils.PIPELINE_ID import com.tencent.devops.process.utils.PROJECT_NAME import com.tencent.devops.process.utils.PipelineVarUtil @@ -150,6 +153,17 @@ open class DefaultModelCheckPlugin constructor( // #4531 检查stage审核组配置是否符合要求 stage.checkStageReviewers() + val stageControlOption = stage.stageControlOption + if (stageControlOption?.runCondition == StageRunCondition.CUSTOM_CONDITION_MATCH) { + val length = stageControlOption.customCondition?.length ?: 0 + if (length > PIPELINE_CONDITION_EXPRESSION_LENGTH_MAX) throw ErrorCodeException( + errorCode = ProcessMessageCode.ERROR_PIPELINE_CONDITION_EXPRESSION_TOO_LONG, + params = arrayOf( + stage.name ?: stage.id.toString(), + PIPELINE_CONDITION_EXPRESSION_LENGTH_MAX.toString() + ) + ) + } val atomVersions = mutableSetOf() val atomInputParamList = mutableListOf() @@ -326,6 +340,15 @@ open class DefaultModelCheckPlugin constructor( addAtomInputDataInfo(element, atomVersions, atomInputParamList) checkElementTimeoutVar(container = this, element = element, contextMap = contextMap) + + val elementControlOption = element.additionalOptions + if (elementControlOption?.runCondition == RunCondition.CUSTOM_CONDITION_MATCH) { + val length = elementControlOption.customCondition?.length ?: 0 + if (length > PIPELINE_CONDITION_EXPRESSION_LENGTH_MAX) throw ErrorCodeException( + errorCode = ProcessMessageCode.ERROR_PIPELINE_CONDITION_EXPRESSION_TOO_LONG, + params = arrayOf(element.name, PIPELINE_CONDITION_EXPRESSION_LENGTH_MAX.toString()) + ) + } } override fun checkElementTimeoutVar(container: Container, element: Element, contextMap: Map) { @@ -556,6 +579,14 @@ open class DefaultModelCheckPlugin constructor( return } + if (jobControlOption.runCondition == JobRunCondition.CUSTOM_CONDITION_MATCH) { + val length = jobControlOption.customCondition?.length ?: 0 + if (length > PIPELINE_CONDITION_EXPRESSION_LENGTH_MAX) throw ErrorCodeException( + errorCode = ProcessMessageCode.ERROR_PIPELINE_CONDITION_EXPRESSION_TOO_LONG, + params = arrayOf(container.name, PIPELINE_CONDITION_EXPRESSION_LENGTH_MAX.toString()) + ) + } + // 非finallyStage下不允许有finallyStageJobRunConditionSet下的条件 if (finallyStage) { if (!finallyStageJobRunConditionSet.contains(jobControlOption.runCondition)) { diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/interceptor/QueueInterceptor.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/interceptor/QueueInterceptor.kt index d34899d6cb7..4df7300c810 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/interceptor/QueueInterceptor.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/interceptor/QueueInterceptor.kt @@ -45,6 +45,7 @@ import com.tencent.devops.process.engine.service.PipelineRuntimeExtService import com.tencent.devops.process.engine.service.PipelineRuntimeService import com.tencent.devops.process.engine.service.PipelineTaskService import com.tencent.devops.common.pipeline.pojo.setting.PipelineRunLockType +import com.tencent.devops.process.constant.ProcessMessageCode import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Component @@ -225,8 +226,12 @@ class QueueInterceptor @Autowired constructor( ) buildLogPrinter.addRedLine( buildId = buildInfo.buildId, - message = "[concurrency] Canceling since " + - "a higher priority waiting request for group($groupName) exists", + message = I18nUtil.getCodeLanMessage( + messageCode = ProcessMessageCode.BK_BUILD_QUEUE_WAIT_FOR_CONCURRENCY, + params = arrayOf(groupName, + "${task.buildId}" + ) + ), tag = "QueueInterceptor", containerHashId = "", executeCount = 1, diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/interceptor/TimerTriggerScmChangeInterceptor.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/interceptor/TimerTriggerScmChangeInterceptor.kt index 524b3df4c55..7a82437905b 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/interceptor/TimerTriggerScmChangeInterceptor.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/interceptor/TimerTriggerScmChangeInterceptor.kt @@ -27,9 +27,9 @@ package com.tencent.devops.process.engine.interceptor +import com.tencent.devops.common.api.enums.CheckoutRepositoryType import com.tencent.devops.common.api.enums.RepositoryConfig import com.tencent.devops.common.api.enums.RepositoryType -import com.tencent.devops.common.api.enums.CheckoutRepositoryType import com.tencent.devops.common.api.util.EnvUtils import com.tencent.devops.common.client.Client import com.tencent.devops.common.pipeline.container.TriggerContainer @@ -102,10 +102,7 @@ class TimerTriggerScmChangeInterceptor @Autowired constructor( return@outer } // 如果插件配置代码库信息,已在PipelineTimerBuildListener已校验源代码是否有变更 - if (!ele.repoHashId.isNullOrBlank() || - !ele.repoName.isNullOrBlank() || - !ele.branches.isNullOrEmpty() - ) { + if (ele.enableRepoConfig()) { noScm = false return@outer } diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/pojo/AgentReuseMutexTree.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/pojo/AgentReuseMutexTree.kt index dcf9db07325..e82620462ff 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/pojo/AgentReuseMutexTree.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/pojo/AgentReuseMutexTree.kt @@ -14,6 +14,8 @@ import com.tencent.devops.common.pipeline.type.agent.ThirdPartyAgentEnvDispatchT import com.tencent.devops.common.pipeline.type.agent.ThirdPartyAgentIDDispatchType import com.tencent.devops.process.constant.ProcessMessageCode import com.tencent.devops.process.engine.control.VmOperateTaskGenerator.Companion.START_VM_TASK_ATOM +import com.tencent.devops.process.pojo.app.StartBuildContext +import com.tencent.devops.process.utils.PIPELINE_NAME /** * AgentReuseMutexTree 组装流水线时通过生成树可以更好地拿到复用互斥关系 @@ -268,6 +270,7 @@ data class AgentReuseMutexTree( } fun rewriteModel( + context: StartBuildContext, buildContainersWithDetail: MutableList>, fullModel: Model, buildTaskList: MutableList @@ -276,7 +279,10 @@ data class AgentReuseMutexTree( buildContainersWithDetail.forEach { (bc, c) -> if (treeMap.containsKey(c.jobId)) { - bc.controlOption.agentReuseMutex = treeMap[c.jobId]?.first + bc.controlOption.agentReuseMutex = treeMap[c.jobId]?.first?.copy( + linkTip = "${context.pipelineId}_Pipeline" + + "[${context.variables[PIPELINE_NAME]}]Job[${c.name}]" + ) } } diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/EngineConfigService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/EngineConfigService.kt new file mode 100644 index 00000000000..ec37b01fc32 --- /dev/null +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/EngineConfigService.kt @@ -0,0 +1,33 @@ +package com.tencent.devops.process.engine.service + +import com.github.benmanes.caffeine.cache.Caffeine +import com.github.benmanes.caffeine.cache.LoadingCache +import com.tencent.devops.common.redis.RedisOperation +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service +import java.time.Duration + +/** + * 流水线引擎动态配置管理类 + * 用来放一些引擎中需要动态配置的参数方法 + */ +@Service +class EngineConfigService @Autowired constructor( + private val redisOperation: RedisOperation +) { + private val localCache: LoadingCache = Caffeine.newBuilder() + .maximumSize(CACHE_SIZE) + .expireAfterWrite(Duration.ofMinutes(CACHE_EXPIRE_MIN)) + .build { key -> redisOperation.get(key, isDistinguishCluster = false) ?: "" } + + fun getMutexMaxQueue() = + localCache.get(ENGINE_CONFIG_MUTEX_MAX_QUEUE_KEY)?.ifBlank { null }?.toIntOrNull() ?: MUTEX_MAX_QUEUE_DEFAULT + + companion object { + private const val ENGINE_CONFIG_MUTEX_MAX_QUEUE_KEY = "engine:config:mutex:maxQueue" + const val MUTEX_MAX_QUEUE_DEFAULT = 50 + + private const val CACHE_EXPIRE_MIN = 10L + private const val CACHE_SIZE = 100L + } +} \ No newline at end of file diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineContainerService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineContainerService.kt index eabad63f7f0..380da9a2f6d 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineContainerService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineContainerService.kt @@ -298,6 +298,7 @@ class PipelineContainerService @Autowired constructor( ): PipelineBuildContainer { var startVMTaskSeq = -1 // 启动构建机位置，解决如果在执行人工审核插件时，无编译环境不需要提前无意义的启动 var taskSeq = 0 + var needStartVM = false // 是否需要启动构建 val parentElements = container.elements parentElements.forEach nextElement@{ atomElement -> @@ -359,23 +360,30 @@ class PipelineContainerService @Autowired constructor( ) } } + // 确认是否要启动构建机/无编译环境 + if (!needStartVM && startVMTaskSeq > 0) { + needStartVM = true + } } container.startVMTaskSeq = startVMTaskSeq // 填入: 构建机或无编译环境的环境处理，需要启动和结束构建机/环境的插件任务 - supplyVMTask( - projectId = projectId, - pipelineId = pipelineId, - buildId = buildId, - userId = context.userId, - stage = stage, - container = container, - containerSeq = context.containerSeq, - startVMTaskSeq = startVMTaskSeq, - lastTimeBuildTasks = listOf(), - updateExistsTask = mutableListOf(), - buildTaskList = buildTaskList, - executeCount = context.executeCount - ) + // 判断是否为无编译环境的写法保持和 prepareBuildContainerTasks() 一致 + if (needStartVM) { + supplyVMTask( + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + userId = context.userId, + stage = stage, + container = container, + containerSeq = context.containerSeq, + startVMTaskSeq = startVMTaskSeq, + lastTimeBuildTasks = listOf(), + updateExistsTask = mutableListOf(), + buildTaskList = buildTaskList, + executeCount = context.executeCount + ) + } return PipelineBuildContainer( projectId = projectId, @@ -384,6 +392,7 @@ class PipelineContainerService @Autowired constructor( stageId = stage.id!!, containerId = container.id!!, containerHashId = container.containerHashId ?: "", + jobId = container.jobId, containerType = container.getClassType(), seq = context.containerSeq, status = BuildStatus.QUEUE, @@ -418,7 +427,7 @@ class PipelineContainerService @Autowired constructor( var needUpdateContainer = false var taskSeq = 0 val containerElements = container.elements - val retryFlag = lastTimeBuildTasks.isEmpty() + val newBuildFlag = lastTimeBuildTasks.isEmpty() containerElements.forEach nextElement@{ atomElement -> modelCheckPlugin.checkElementTimeoutVar(container, atomElement, contextMap = context.variables) @@ -440,7 +449,7 @@ class PipelineContainerService @Autowired constructor( if (status.isFinish()) { logger.info("[${context.buildId}|${atomElement.id}] status=$status") atomElement.status = status.name - if (retryFlag && ElementUtils.getTaskAddFlag( + if (newBuildFlag && ElementUtils.getTaskAddFlag( element = atomElement, stageEnableFlag = stage.isStageEnable(), containerEnableFlag = container.isContainerEnable(), @@ -465,7 +474,7 @@ class PipelineContainerService @Autowired constructor( } // 全新构建，其中构建矩阵不需要添加待执行插件 - if (retryFlag) { + if (newBuildFlag) { if (container.matrixGroupFlag != true) { context.taskCount++ val buildTask = genBuildTaskToList( @@ -561,8 +570,8 @@ class PipelineContainerService @Autowired constructor( } container.startVMTaskSeq = startVMTaskSeq - // 构建矩阵没有对应的重试插件，单独增加重试记录 - if (context.needRerunStage(stage = stage) && container.matrixGroupFlag == true) { + // 构建矩阵永远跟随stage重试，在需要重试的stage中，单独增加重试记录 + if (container.matrixGroupFlag == true && !context.needSkipContainerWhenFailRetry(stage, container)) { container.retryFreshMatrixOption() cleanContainersInMatrixGroup( transactionContext = dslContext, @@ -595,9 +604,9 @@ class PipelineContainerService @Autowired constructor( ) } logger.info( - "prepareBuildContainerTasks|buildId=${context.buildId}|matrixGroupFlag=${container.matrixGroupFlag}|" + - "needUpdateContainer=$needUpdateContainer|needStartVM=$needStartVM|" + - "startVMTaskSeq=${container.startVMTaskSeq}" + "prepareBuildContainerTasks|buildId=${context.buildId}|containerId=${container.id}|" + + "matrixGroupFlag=${container.matrixGroupFlag}|needUpdateContainer=$needUpdateContainer|" + + "needStartVM=$needStartVM|startVMTaskSeq=$startVMTaskSeq|" ) if (needUpdateContainer) { container.resetBuildOption(context.executeCount) @@ -654,6 +663,7 @@ class PipelineContainerService @Autowired constructor( stageId = stage.id!!, containerId = container.id!!, containerHashId = container.containerHashId ?: "", + jobId = container.jobId, containerType = container.getClassType(), seq = context.containerSeq, status = BuildStatus.QUEUE, @@ -931,7 +941,8 @@ class PipelineContainerService @Autowired constructor( subProjectId = null, subBuildId = null, atomCode = atomElement.getAtomCode(), - stepId = atomElement.stepId + stepId = atomElement.stepId, + jobId = container.jobId ) fun setUpTriggerContainer( diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRepositoryService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRepositoryService.kt index 97104e84ada..9a6509046a8 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRepositoryService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRepositoryService.kt @@ -38,6 +38,7 @@ import com.tencent.devops.common.api.util.MessageUtil import com.tencent.devops.common.api.util.Watcher import com.tencent.devops.common.audit.ActionAuditContent import com.tencent.devops.common.client.Client +import com.tencent.devops.common.db.utils.JooqUtils import com.tencent.devops.common.event.dispatcher.pipeline.PipelineEventDispatcher import com.tencent.devops.common.event.pojo.pipeline.PipelineModelAnalysisEvent import com.tencent.devops.common.pipeline.Model @@ -101,9 +102,11 @@ import com.tencent.devops.process.pojo.PipelineSortType import com.tencent.devops.process.pojo.pipeline.DeletePipelineResult import com.tencent.devops.process.pojo.pipeline.DeployPipelineResult import com.tencent.devops.process.pojo.pipeline.PipelineResourceVersion +import com.tencent.devops.process.pojo.pipeline.PipelineYamlVo import com.tencent.devops.process.pojo.pipeline.TemplateInfo import com.tencent.devops.process.pojo.setting.PipelineModelVersion import com.tencent.devops.process.service.PipelineOperationLogService +import com.tencent.devops.process.service.label.PipelineGroupService import com.tencent.devops.process.service.pipeline.PipelineSettingVersionService import com.tencent.devops.process.service.pipeline.PipelineTransferYamlService import com.tencent.devops.process.utils.PIPELINE_MATRIX_CON_RUNNING_SIZE_MAX @@ -158,7 +161,8 @@ class PipelineRepositoryService constructor( private val client: Client, private val transferService: PipelineTransferYamlService, private val redisOperation: RedisOperation, - private val pipelineYamlInfoDao: PipelineYamlInfoDao + private val pipelineYamlInfoDao: PipelineYamlInfoDao, + private val pipelineGroupService: PipelineGroupService ) { companion object { @@ -224,7 +228,8 @@ class PipelineRepositoryService constructor( setting: PipelineSetting? = null, versionStatus: VersionStatus? = VersionStatus.RELEASED, branchName: String? = null, - description: String? = null + description: String? = null, + yamlInfo: PipelineYamlVo? = null ): DeployPipelineResult { // 生成流水线ID,新流水线以p-开头，以区分以前旧数据 @@ -237,7 +242,8 @@ class PipelineRepositoryService constructor( userId = userId, create = create, versionStatus = versionStatus, - channelCode = channelCode + channelCode = channelCode, + yamlInfo = yamlInfo ) val buildNo = (model.stages[0].containers[0] as TriggerContainer).buildNo @@ -281,27 +287,29 @@ class PipelineRepositoryService constructor( ) result } else { - val result = create( - projectId = projectId, - pipelineId = pipelineId, - model = model, - customSetting = setting, - yaml = yaml, - userId = userId, - channelCode = channelCode, - canManualStartup = canManualStartup, - canElementSkip = canElementSkip, - buildNo = buildNo, - modelTasks = modelTasks, - useSubscriptionSettings = useSubscriptionSettings, - useLabelSettings = useLabelSettings, - useConcurrencyGroup = useConcurrencyGroup, - templateId = templateId, - versionStatus = versionStatus, - branchName = branchName, - description = description, - baseVersion = baseVersion - ) + val result = JooqUtils.retryWhenDeadLock(3) { + create( + projectId = projectId, + pipelineId = pipelineId, + model = model, + customSetting = setting, + yaml = yaml, + userId = userId, + channelCode = channelCode, + canManualStartup = canManualStartup, + canElementSkip = canElementSkip, + buildNo = buildNo, + modelTasks = modelTasks, + useSubscriptionSettings = useSubscriptionSettings, + useLabelSettings = useLabelSettings, + useConcurrencyGroup = useConcurrencyGroup, + templateId = templateId, + versionStatus = versionStatus, + branchName = branchName, + description = description, + baseVersion = baseVersion + ) + } operationLogService.addOperationLog( userId = userId, projectId = projectId, @@ -327,7 +335,8 @@ class PipelineRepositoryService constructor( userId: String, create: Boolean = true, versionStatus: VersionStatus? = VersionStatus.RELEASED, - channelCode: ChannelCode + channelCode: ChannelCode, + yamlInfo: PipelineYamlVo? = null ): List { val metaSize = modelCheckPlugin.checkModelIntegrity(model, projectId, userId) @@ -355,7 +364,8 @@ class PipelineRepositoryService constructor( channelCode = channelCode, create = create, distIds = distinctIdSet, - versionStatus = versionStatus + versionStatus = versionStatus, + yamlInfo = yamlInfo ) } else { initOtherContainer( @@ -369,7 +379,8 @@ class PipelineRepositoryService constructor( channelCode = channelCode, create = create, distIds = distinctIdSet, - versionStatus = versionStatus + versionStatus = versionStatus, + yamlInfo = yamlInfo ) } } @@ -388,7 +399,8 @@ class PipelineRepositoryService constructor( channelCode: ChannelCode, create: Boolean, distIds: HashSet, - versionStatus: VersionStatus? = VersionStatus.RELEASED + versionStatus: VersionStatus? = VersionStatus.RELEASED, + yamlInfo: PipelineYamlVo? ) { if (stage.containers.size != 1) { logger.warn("The trigger stage contain more than one container (${stage.containers.size})") @@ -429,7 +441,8 @@ class PipelineRepositoryService constructor( userId = userId, channelCode = channelCode, create = create, - container = c + container = c, + yamlInfo = yamlInfo ) } @@ -464,7 +477,8 @@ class PipelineRepositoryService constructor( channelCode: ChannelCode, create: Boolean, distIds: HashSet, - versionStatus: VersionStatus? = VersionStatus.RELEASED + versionStatus: VersionStatus? = VersionStatus.RELEASED, + yamlInfo: PipelineYamlVo? ) { if (stage.containers.isEmpty()) { throw ErrorCodeException( @@ -549,7 +563,8 @@ class PipelineRepositoryService constructor( userId = userId, channelCode = channelCode, create = create, - container = c + container = c, + yamlInfo = yamlInfo ) } @@ -656,6 +671,7 @@ class PipelineRepositoryService constructor( pipelineName = model.name, desc = model.desc ?: "" ) ?: run { + // 空白流水线设置初始化 val maxPipelineResNum = if ( channelCode.name in versionConfigure.specChannels.split(",") ) { @@ -683,28 +699,24 @@ class PipelineRepositoryService constructor( if (model.instanceFromTemplate != true) { if (null == pipelineSettingDao.getSetting(transactionContext, projectId, pipelineId)) { - if (templateId != null && (useSubscriptionSettings == true || useConcurrencyGroup == true)) { + if (useTemplateSettings( + templateId = templateId, + useSubscriptionSettings = useSubscriptionSettings, + useLabelSettings = useLabelSettings, + useConcurrencyGroup = useConcurrencyGroup + ) + ) { // 沿用模板的配置 - val setting = getSetting(projectId, templateId) + val setting = getSetting(projectId, templateId!!) ?: throw ErrorCodeException(errorCode = ProcessMessageCode.PIPELINE_SETTING_NOT_EXISTS) setting.pipelineId = pipelineId setting.pipelineName = model.name setting.version = settingVersion if (useSubscriptionSettings != true) { - setting.successSubscription = Subscription( - types = setOf(), - groups = emptySet(), - users = "\${{ci.actor}}", - content = NotifyTemplateUtils.getCommonShutdownSuccessContent() - ) + setting.successSubscription = null setting.successSubscriptionList = null - setting.failSubscription = Subscription( - types = setOf(PipelineSubscriptionType.EMAIL, PipelineSubscriptionType.RTX), - groups = emptySet(), - users = "\${{ci.actor}}", - content = NotifyTemplateUtils.getCommonShutdownFailureContent() - ) - setting.failSubscriptionList = listOf(setting.failSubscription) + setting.failSubscription = null + setting.failSubscriptionList = newSetting.failSubscriptionList } if (useConcurrencyGroup != true) { setting.concurrencyGroup = null @@ -713,6 +725,13 @@ class PipelineRepositoryService constructor( } if (useLabelSettings != true) { setting.labels = listOf() + } else { + val groups = pipelineGroupService.getGroups(userId, projectId, templateId) + val labels = ArrayList() + groups.forEach { + labels.addAll(it.labels) + } + setting.labels = labels } setting.pipelineAsCodeSettings = PipelineAsCodeSettings() newSetting = setting @@ -738,7 +757,8 @@ class PipelineRepositoryService constructor( dslContext = transactionContext, setting = setting, id = client.get(ServiceAllocIdResource::class) - .generateSegmentId(PIPELINE_SETTING_VERSION_BIZ_TAG_NAME).data, + .generateSegmentId(PIPELINE_SETTING_VERSION_BIZ_TAG_NAME) + .data, version = settingVersion ) newSetting = setting @@ -774,7 +794,7 @@ class PipelineRepositoryService constructor( dslContext = transactionContext, projectId = projectId, pipelineId = pipelineId, - creator = userId, + userId = userId, version = 1, model = model, yamlStr = yaml?.yamlStr, @@ -825,6 +845,16 @@ class PipelineRepositoryService constructor( ) } + private fun useTemplateSettings( + templateId: String? = null, + useSubscriptionSettings: Boolean? = false, + useLabelSettings: Boolean? = false, + useConcurrencyGroup: Boolean? = false + ): Boolean { + return templateId != null && + (useSubscriptionSettings == true || useConcurrencyGroup == true || useLabelSettings == true) + } + private fun update( projectId: String, pipelineId: String, @@ -1081,7 +1111,7 @@ class PipelineRepositoryService constructor( dslContext = transactionContext, projectId = projectId, pipelineId = pipelineId, - creator = userId, + userId = userId, version = version, model = model, yamlStr = yaml?.yamlStr, @@ -1168,18 +1198,17 @@ class PipelineRepositoryService constructor( dslContext = transactionContext, projectId = projectId, pipelineId = pipelineId, - creator = userId, + userId = userId, version = releaseResource.version, model = releaseResource.model, yamlVersion = releaseResource.yamlVersion, yamlStr = releaseResource.yaml, baseVersion = baseVersion, - versionName = releaseResource.versionName ?: PipelineVersionUtils.getVersionName( - releaseResource.version, releaseResource.version, 0, 0 - ) ?: "", + versionName = releaseResource.versionName ?: "init", versionNum = releaseResource.versionNum, - pipelineVersion = null, + pipelineVersion = releaseResource.version, triggerVersion = null, + // 不写入版本状态和关联的setting版本，标识是兼容非常老的数据补全 settingVersion = null, versionStatus = null, branchAction = null, @@ -1238,13 +1267,20 @@ class PipelineRepositoryService constructor( ).map { it.key to str2model(it.value, it.key) }.toMap() } + /** + * 获取编排版本的通用方法 + * 1 如果指定了[version]则一定按照version号查询版本 + * 2 如果没有指定版本，则通过[includeDraft]控制是否过滤掉草稿，获得最新版本流水线 + * 3 默认情况，[version]=null 且 [includeDraft]=false 时，直接返回当前正式版本 + */ fun getPipelineResourceVersion( projectId: String, pipelineId: String, version: Int? = null, includeDraft: Boolean? = false ): PipelineResourceVersion? { - val resource = if (version == null) { // 取最新版，直接从旧版本表读 + // TODO 取不到则直接从旧版本表读，待下架 + val resource = if (version == null) { if (includeDraft == true) pipelineResourceVersionDao.getDraftVersionResource( dslContext = dslContext, projectId = projectId, @@ -1313,7 +1349,7 @@ class PipelineRepositoryService constructor( fun getBranchVersionResource( projectId: String, pipelineId: String, - branchName: String + branchName: String? ): PipelineResourceVersion? { val resource = pipelineResourceVersionDao.getBranchVersionResource( dslContext = dslContext, @@ -1391,7 +1427,7 @@ class PipelineRepositoryService constructor( val now = LocalDateTime.now() val newDraft = targetVersion.copy( version = latestResource.version + 1, - versionNum = releaseResource.version + 1, + versionNum = null, pipelineVersion = null, triggerVersion = null, versionName = null, @@ -1404,7 +1440,7 @@ class PipelineRepositoryService constructor( dslContext = context, projectId = projectId, pipelineId = pipelineId, - creator = userId, + userId = userId, version = newDraft.version, versionName = "", model = newDraft.model, @@ -1468,7 +1504,6 @@ class PipelineRepositoryService constructor( if (deleteName.length > MAX_LEN_FOR_NAME) { // 超过截断，且用且珍惜 deleteName = deleteName.substring(0, MAX_LEN_FOR_NAME) } - pipelineResourceVersionDao.clearDraftVersion(transactionContext, projectId, pipelineId) pipelineResourceVersionDao.clearActiveBranchVersion(transactionContext, projectId, pipelineId) pipelineInfoDao.softDelete( dslContext = transactionContext, @@ -1672,6 +1707,28 @@ class PipelineRepositoryService constructor( ) } + return JooqUtils.retryWhenDeadLock(3) { + transactionSaveSetting( + context = context, + setting = setting, + versionStatus = versionStatus, + userId = userId, + updateLastModifyUser = updateLastModifyUser, + version = version, + isTemplate = isTemplate + ) + } + } + + private fun transactionSaveSetting( + context: DSLContext?, + setting: PipelineSetting, + versionStatus: VersionStatus, + userId: String, + updateLastModifyUser: Boolean?, + version: Int, + isTemplate: Boolean + ): PipelineName { var oldName: String = setting.pipelineName (context ?: dslContext).transaction { t -> val transactionContext = DSL.using(t) @@ -1714,9 +1771,11 @@ class PipelineRepositoryService constructor( ).data ) } - if (versionStatus.isReleasing()) pipelineSettingDao.saveSetting( - transactionContext, setting, isTemplate - ).toString() + if (versionStatus.isReleasing()) { + pipelineSettingDao.saveSetting( + transactionContext, setting, isTemplate + ) + } } return PipelineName(name = setting.pipelineName, oldName = oldName) @@ -1965,11 +2024,8 @@ class PipelineRepositoryService constructor( ) ).yamlWithVersion } catch (ignore: Throwable) { - if (ignore is ErrorCodeException) throw ignore logger.warn("TRANSFER_YAML_SETTING|$projectId|$userId|$pipelineId", ignore) - throw ErrorCodeException( - errorCode = ProcessMessageCode.ERROR_OCCURRED_IN_TRANSFER - ) + null } watcher.start("updatePipelineInfo") pipelineInfoDao.update( @@ -2008,7 +2064,7 @@ class PipelineRepositoryService constructor( dslContext = transactionContext, projectId = projectId, pipelineId = pipelineId, - creator = userId, + userId = userId, version = version, model = newModel, yamlStr = yamlWithVersion?.yamlStr, @@ -2040,6 +2096,7 @@ class PipelineRepositoryService constructor( } fun updatePipelineBranchVersion( + userId: String, projectId: String, pipelineId: String, branchName: String?, @@ -2048,10 +2105,27 @@ class PipelineRepositoryService constructor( ) { pipelineResourceVersionDao.updateBranchVersion( dslContext = transactionContext ?: dslContext, + userId = userId, projectId = projectId, pipelineId = pipelineId, branchName = branchName, branchVersionAction = branchVersionAction ) } + + fun updateLocked( + userId: String, + projectId: String, + pipelineId: String, + locked: Boolean, + transactionContext: DSLContext? = null + ): Boolean { + return pipelineInfoDao.update( + dslContext = transactionContext ?: dslContext, + userId = userId, + projectId = projectId, + pipelineId = pipelineId, + locked = locked + ) + } } diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRepositoryVersionService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRepositoryVersionService.kt index 32e9b33265e..5c0c570afab 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRepositoryVersionService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRepositoryVersionService.kt @@ -201,6 +201,85 @@ class PipelineRepositoryVersionService( versionName: String?, creator: String?, description: String? + ): Pair> { + if (pipelineInfo == null) { + return Pair(0, mutableListOf()) + } + + var count = pipelineResourceVersionDao.count( + dslContext = dslContext, + projectId = projectId, + pipelineId = pipelineId, + includeDraft = includeDraft, + versionName = versionName, + creator = creator, + description = description + ) + val result = mutableListOf() + result.addAll( + pipelineResourceVersionDao.listPipelineVersion( + dslContext = dslContext, + projectId = projectId, + pipelineId = pipelineId, + pipelineInfo = pipelineInfo, + creator = creator, + description = description, + versionName = versionName, + includeDraft = includeDraft, + excludeVersion = excludeVersion, + offset = offset, + limit = limit + ) + ) + // #8161 当过滤草稿时查到空结果是正常的，只在不过滤草稿时兼容老数据的版本表无记录 + if (result.isEmpty() && pipelineInfo.latestVersionStatus?.isNotReleased() != true) { + pipelineResourceDao.getReleaseVersionResource( + dslContext, projectId, pipelineId + )?.let { record -> + count = 1 + result.add( + PipelineVersionSimple( + pipelineId = record.pipelineId, + creator = record.creator, + createTime = record.createTime.timestampmilli(), + updater = record.updater, + updateTime = record.updateTime?.timestampmilli(), + version = record.version, + versionName = record.versionName ?: PipelineVersionUtils.getVersionName( + versionNum = record.version, + pipelineVersion = record.versionNum ?: record.version, + triggerVersion = 0, + settingVersion = 0 + ) ?: "", + yamlVersion = record.yamlVersion, + referFlag = record.referFlag, + referCount = record.referCount, + versionNum = record.versionNum ?: record.version, + pipelineVersion = record.pipelineVersion, + triggerVersion = record.triggerVersion, + settingVersion = record.settingVersion, + status = record.status, + debugBuildId = record.debugBuildId, + baseVersion = record.baseVersion, + description = record.description + ) + ) + } + } + return count to result + } + + fun listPipelineVersionWithInfo( + pipelineInfo: PipelineInfo?, + projectId: String, + pipelineId: String, + offset: Int, + limit: Int, + includeDraft: Boolean?, + excludeVersion: Int?, + versionName: String?, + creator: String?, + description: String? ): Pair> { if (pipelineInfo == null) { return Pair(0, mutableListOf()) @@ -221,6 +300,7 @@ class PipelineRepositoryVersionService( dslContext = dslContext, projectId = projectId, pipelineId = pipelineId, + pipelineInfo = pipelineInfo, creator = creator, description = description, versionName = versionName, @@ -240,6 +320,7 @@ class PipelineRepositoryVersionService( PipelineVersionSimple( pipelineId = record.pipelineId, creator = record.creator, + updater = record.updater, createTime = record.createTime.timestampmilli(), updateTime = record.updateTime?.timestampmilli(), version = record.version, @@ -382,6 +463,9 @@ class PipelineRepositoryVersionService( private fun updatePipelineReferFlag(projectId: String, pipelineId: String) { var offset = 0 val limit = PageUtil.DEFAULT_PAGE_SIZE + val pipelineInfo = pipelineInfoDao.convert( + pipelineInfoDao.getPipelineInfo(dslContext, projectId, pipelineId), null + ) ?: return val lock = PipelineModelLock(redisOperation, pipelineId) try { lock.lock() @@ -391,6 +475,7 @@ class PipelineRepositoryVersionService( dslContext = dslContext, projectId = projectId, pipelineId = pipelineId, + pipelineInfo = pipelineInfo, queryUnknownRelatedFlag = true, offset = offset, limit = limit diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRuntimeService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRuntimeService.kt index 5a608921e74..86a80d22fe2 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRuntimeService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineRuntimeService.kt @@ -106,6 +106,7 @@ import com.tencent.devops.process.engine.service.record.TaskBuildRecordService import com.tencent.devops.process.engine.service.rule.PipelineRuleService import com.tencent.devops.process.engine.utils.ContainerUtils import com.tencent.devops.process.engine.utils.PipelineUtils +import com.tencent.devops.process.enums.HistorySearchType import com.tencent.devops.process.pojo.BuildBasicInfo import com.tencent.devops.process.pojo.BuildHistory import com.tencent.devops.process.pojo.BuildId @@ -197,7 +198,8 @@ class PipelineRuntimeService @Autowired constructor( statusSet = setOf( BuildStatus.RUNNING, BuildStatus.REVIEWING, BuildStatus.QUEUE, BuildStatus.PREPARE_ENV, - BuildStatus.UNEXEC, BuildStatus.QUEUE_CACHE + BuildStatus.UNEXEC, BuildStatus.QUEUE_CACHE, + BuildStatus.STAGE_SUCCESS ) ) @@ -370,7 +372,9 @@ class PipelineRuntimeService @Autowired constructor( startUser: List?, updateTimeDesc: Boolean? = null, queryDslContext: DSLContext? = null, - debugVersion: Int? + debugVersion: Int?, + triggerAlias: List?, + triggerBranch: List? ): List { val currentTimestamp = System.currentTimeMillis() // 限制最大一次拉1000，防止攻击 @@ -403,7 +407,9 @@ class PipelineRuntimeService @Autowired constructor( buildMsg = buildMsg, startUser = startUser, updateTimeDesc = updateTimeDesc, - debugVersion = debugVersion + debugVersion = debugVersion, + triggerAlias = triggerAlias, + triggerBranch = triggerBranch ) val result = mutableListOf() list.forEach { @@ -416,43 +422,124 @@ class PipelineRuntimeService @Autowired constructor( pipelineBuildDao.updateBuildRemark(dslContext, projectId, pipelineId, buildId, remark) } - fun getHistoryConditionRepo(projectId: String, pipelineId: String, debugVersion: Int?): List { - val history = pipelineBuildDao.getBuildHistoryMaterial(dslContext, projectId, pipelineId, debugVersion) - val materialObjList = mutableListOf() - history.forEach { - if (!it.material.isNullOrEmpty()) { - materialObjList.addAll(it.material!!) + fun getHistoryConditionRepo( + projectId: String, + pipelineId: String, + debugVersion: Int?, + search: String?, + type: HistorySearchType? = HistorySearchType.MATERIAL + ): List { + val aliasNames = when (type) { + HistorySearchType.MATERIAL -> { + val history = pipelineBuildDao.listHistorySearchOptions( + dslContext = dslContext, + projectId = projectId, + pipelineId = pipelineId, + debugVersion = debugVersion, + type = type + ) + val materialObjList = mutableListOf() + history.forEach { + if (!it.material.isNullOrEmpty()) { + materialObjList.addAll(it.material!!) + } + } + materialObjList.filter { !it.aliasName.isNullOrBlank() } + .map { it.aliasName!! } + .distinct() + } + + HistorySearchType.TRIGGER -> { + val history = pipelineBuildDao.listHistorySearchOptions( + dslContext = dslContext, + projectId = projectId, + pipelineId = pipelineId, + debugVersion = debugVersion, + type = type + ) + history.filter { it.webhookInfo != null && !it.webhookInfo!!.webhookAliasName.isNullOrBlank() } + .map { it.webhookInfo!!.webhookAliasName!! } + .distinct() } + + else -> emptyList() + } + return if (search.isNullOrBlank()) { + aliasNames + } else { + aliasNames.filter { it.contains(search) } } - return materialObjList.filter { !it.aliasName.isNullOrBlank() }.map { it.aliasName!! }.distinct() } fun getHistoryConditionBranch( projectId: String, pipelineId: String, aliasList: List?, - debugVersion: Int? = null + debugVersion: Int? = null, + search: String?, + type: HistorySearchType? = HistorySearchType.MATERIAL ): List { - val history = pipelineBuildDao.getBuildHistoryMaterial(dslContext, projectId, pipelineId, debugVersion) - val materialObjList = mutableListOf() - history.forEach { - if (!it.material.isNullOrEmpty()) { - materialObjList.addAll(it.material!!) + val branchNames = when (type) { + HistorySearchType.MATERIAL -> { + val history = pipelineBuildDao.listHistorySearchOptions( + dslContext = dslContext, + projectId = projectId, + pipelineId = pipelineId, + debugVersion = debugVersion, + type = type + ) + val materialObjList = mutableListOf() + history.forEach { + if (!it.material.isNullOrEmpty()) { + materialObjList.addAll(it.material!!) + } + } + val aliasNames = if (!aliasList.isNullOrEmpty() && aliasList.first().isNotBlank()) { + aliasList + } else { + materialObjList.map { it.aliasName } + } + + val result = mutableListOf() + aliasNames.distinct().forEach { alias -> + val branchNames = materialObjList.filter { + it.aliasName == alias && !it.branchName.isNullOrBlank() + }.map { it.branchName!! }.distinct() + result.addAll(branchNames) + } + result.distinct() } + + HistorySearchType.TRIGGER -> { + val history = pipelineBuildDao.listHistorySearchOptions( + dslContext = dslContext, + projectId = projectId, + pipelineId = pipelineId, + debugVersion = debugVersion, + type = type + ) + val webhookInfoList = history.filter { it.webhookInfo != null }.map { it.webhookInfo!! } + val aliasNames = if (!aliasList.isNullOrEmpty() && aliasList.first().isNotBlank()) { + aliasList + } else { + webhookInfoList.map { it.webhookAliasName } + } + val result = mutableListOf() + aliasNames.distinct().forEach { alias -> + val branchNames = webhookInfoList.filter { + it.webhookAliasName == alias && !it.webhookBranch.isNullOrBlank() + }.map { it.webhookBranch!! }.distinct() + result.addAll(branchNames) + } + result.distinct() + } + else -> emptyList() } - val aliasNames = if (aliasList.isNullOrEmpty()) { - materialObjList.map { it.aliasName } + return if (search.isNullOrBlank()) { + branchNames } else { - aliasList - } - - val result = mutableListOf() - aliasNames.distinct().forEach { alias -> - val branchNames = materialObjList.filter { it.aliasName == alias && !it.branchName.isNullOrBlank() } - .map { it.branchName!! }.distinct() - result.addAll(branchNames) + branchNames.filter { it.contains(search) } } - return result.distinct() } private fun genBuildHistory( @@ -481,7 +568,9 @@ class PipelineRuntimeService @Autowired constructor( queueTime = queueTime, artifactList = artifactList, remark = remark, - totalTime = startTime?.let { s -> endTime?.let { e -> e - s } ?: 0 } ?: 0, + totalTime = if (startTime != null && endTime != null) { + (endTime!! - startTime!!).takeIf { it > 0 } + } else null, executeTime = executeTime, buildParameters = buildParameters, webHookType = webhookType, @@ -503,9 +592,12 @@ class PipelineRuntimeService @Autowired constructor( projectId: String, pipelineId: String, buildNum: Int?, - statusSet: Set? + statusSet: Set?, + debug: Boolean? = false ): BuildHistory? { - val record = pipelineBuildDao.getBuildInfoByBuildNum(dslContext, projectId, pipelineId, buildNum, statusSet) + val record = pipelineBuildDao.getBuildInfoByBuildNum( + dslContext, projectId, pipelineId, buildNum, statusSet, debug ?: false + ) return if (record != null) { genBuildHistory(record, System.currentTimeMillis()) } else { @@ -521,11 +613,11 @@ class PipelineRuntimeService @Autowired constructor( } buildIds.forEach { buildId -> - result[buildId] = BuildBasicInfo(buildId, "", "", 0) + result[buildId] = BuildBasicInfo(buildId, "", "", 0, null) } records.forEach { with(it) { - result[it.buildId] = BuildBasicInfo(buildId, projectId, pipelineId, version) + result[it.buildId] = BuildBasicInfo(buildId, projectId, pipelineId, version, status) } } return result @@ -880,7 +972,7 @@ class PipelineRuntimeService @Autowired constructor( } // #10082 使用互斥树节点重新回写Control和Container - agentReuseMutexTree.rewriteModel(buildContainersWithDetail, fullModel, buildTaskList) + agentReuseMutexTree.rewriteModel(context, buildContainersWithDetail, fullModel, buildTaskList) context.pipelineParamMap[PIPELINE_START_TASK_ID] = BuildParameters(PIPELINE_START_TASK_ID, context.firstTaskId, readOnly = true) @@ -1297,7 +1389,7 @@ class PipelineRuntimeService @Autowired constructor( private fun StartBuildContext.sendBuildStartEvent() { // #8275 在发送运行或排队的开始事件时，进行排队计数+1 - if (!debug) pipelineBuildSummaryDao.updateQueueCount( + pipelineBuildSummaryDao.updateQueueCount( dslContext = dslContext, projectId = projectId, pipelineId = pipelineId, @@ -1585,11 +1677,11 @@ class PipelineRuntimeService @Autowired constructor( pipelineId = latestRunningBuild.pipelineId, startTime = startTime ) - // #8161 调试构建不参与刷新summary表 - if (!latestRunningBuild.debug) pipelineBuildSummaryDao.startLatestRunningBuild( + pipelineBuildSummaryDao.startLatestRunningBuild( dslContext = transactionContext, latestRunningBuild = latestRunningBuild, - executeCount = latestRunningBuild.executeCount + executeCount = latestRunningBuild.executeCount, + debug = latestRunningBuild.debug ) } pipelineEventDispatcher.dispatch( @@ -1621,20 +1713,20 @@ class PipelineRuntimeService @Autowired constructor( errorInfoList: List?, timeCost: BuildRecordTimeCost? ) { - if (!latestRunningBuild.debug) { - if (currentBuildStatus.isReadyToRun() || currentBuildStatus.isNeverRun()) { - // 减1,当作没执行过 - pipelineBuildSummaryDao.updateQueueCount( - dslContext = dslContext, - projectId = latestRunningBuild.projectId, - pipelineId = latestRunningBuild.pipelineId, - queueIncrement = -1 - ) - } + if (currentBuildStatus.isReadyToRun() || currentBuildStatus.isNeverRun()) { + // 减1,当作没执行过 + pipelineBuildSummaryDao.updateQueueCount( + dslContext = dslContext, + projectId = latestRunningBuild.projectId, + pipelineId = latestRunningBuild.pipelineId, + queueIncrement = -1 + ) + } else { pipelineBuildSummaryDao.finishLatestRunningBuild( dslContext = dslContext, latestRunningBuild = latestRunningBuild, - isStageFinish = currentBuildStatus.name == BuildStatus.STAGE_SUCCESS.name + isStageFinish = currentBuildStatus.name == BuildStatus.STAGE_SUCCESS.name, + debug = latestRunningBuild.debug ) } with(latestRunningBuild) { @@ -1786,7 +1878,9 @@ class PipelineRuntimeService @Autowired constructor( buildMsg: String? = null, startUser: List? = null, queryDslContext: DSLContext? = null, - debugVersion: Int? = null + debugVersion: Int? = null, + triggerAlias: List?, + triggerBranch: List? ): Int { return pipelineBuildDao.count( dslContext = queryDslContext ?: dslContext, @@ -1812,7 +1906,9 @@ class PipelineRuntimeService @Autowired constructor( buildNoEnd = buildNoEnd, buildMsg = buildMsg, startUser = startUser, - debugVersion = debugVersion + debugVersion = debugVersion, + triggerAlias = triggerAlias, + triggerBranch = triggerBranch ) } @@ -2010,4 +2106,36 @@ class PipelineRuntimeService @Autowired constructor( redisLock.unlock() } } + + fun updateAsyncStatus( + projectId: String, + pipelineId: String, + buildId: String, + taskId: String, + executeCount: Int, + asyncStatus: String + ) { + taskBuildRecordService.updateAsyncStatus( + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + taskId = taskId, + executeCount = executeCount, + asyncStatus = asyncStatus + ) + } + + fun getBuildVariableService( + projectId: String, + pipelineId: String, + buildId: String, + keys: Set + ): Map { + return buildVariableService.getAllVariable( + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + keys = keys + ) + } } diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineStageService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineStageService.kt index f7bdb074582..043cda23c1a 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineStageService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/PipelineStageService.kt @@ -30,6 +30,7 @@ package com.tencent.devops.process.engine.service import com.tencent.devops.common.api.enums.BuildReviewType import com.tencent.devops.common.api.util.DateTimeUtil import com.tencent.devops.common.client.Client +import com.tencent.devops.common.db.utils.JooqUtils import com.tencent.devops.common.event.dispatcher.pipeline.PipelineEventDispatcher import com.tencent.devops.common.event.enums.ActionType import com.tencent.devops.common.event.pojo.pipeline.PipelineBuildQualityCheckBroadCastEvent @@ -39,7 +40,6 @@ import com.tencent.devops.common.pipeline.enums.BuildStatus import com.tencent.devops.common.pipeline.enums.ManualReviewAction import com.tencent.devops.common.pipeline.pojo.StagePauseCheck import com.tencent.devops.common.pipeline.pojo.StageReviewRequest -import com.tencent.devops.common.db.utils.JooqUtils import com.tencent.devops.common.websocket.enum.RefreshType import com.tencent.devops.process.engine.common.BS_MANUAL_START_STAGE import com.tencent.devops.process.engine.common.BS_QUALITY_ABORT_STAGE @@ -53,7 +53,6 @@ import com.tencent.devops.process.engine.pojo.PipelineBuildStage import com.tencent.devops.process.engine.pojo.event.PipelineBuildNotifyEvent import com.tencent.devops.process.engine.pojo.event.PipelineBuildStageEvent import com.tencent.devops.process.engine.pojo.event.PipelineBuildWebSocketPushEvent -import com.tencent.devops.process.engine.service.detail.StageBuildDetailService import com.tencent.devops.process.engine.service.record.StageBuildRecordService import com.tencent.devops.process.pojo.PipelineNotifyTemplateEnum import com.tencent.devops.process.pojo.StageQualityRequest @@ -65,12 +64,12 @@ import com.tencent.devops.process.utils.PipelineVarUtil import com.tencent.devops.quality.api.v2.pojo.ControlPointPosition import com.tencent.devops.quality.api.v3.ServiceQualityRuleResource import com.tencent.devops.quality.api.v3.pojo.request.BuildCheckParamsV3 +import java.util.Date import org.jooq.DSLContext import org.jooq.impl.DSL import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service -import java.util.Date /** * 流水线Stage相关的服务 @@ -85,7 +84,6 @@ class PipelineStageService @Autowired constructor( private val pipelineBuildSummaryDao: PipelineBuildSummaryDao, private val pipelineBuildStageDao: PipelineBuildStageDao, private val buildVariableService: BuildVariableService, - private val stageBuildDetailService: StageBuildDetailService, private val stageBuildRecordService: StageBuildRecordService, private val pipelineRepositoryService: PipelineRepositoryService, private val client: Client @@ -215,7 +213,8 @@ class PipelineStageService @Autowired constructor( with(buildStage) { // 兜底保护，若已经被审核过则直接忽略 if (checkIn?.status == BuildStatus.REVIEW_ABORT.name || - checkIn?.status == BuildStatus.REVIEW_PROCESSED.name) { + checkIn?.status == BuildStatus.REVIEW_PROCESSED.name + ) { return@with } checkIn?.status = BuildStatus.REVIEWING.name @@ -246,9 +245,9 @@ class PipelineStageService @Autowired constructor( dslContext = context, projectId = projectId, buildId = buildId, stageStatus = allStageStatus ) // 被暂停的流水线不占构建队列，在执行数-1 - if (!debug) pipelineBuildSummaryDao.updateRunningCount( + pipelineBuildSummaryDao.updateRunningCount( dslContext = context, projectId = projectId, pipelineId = pipelineId, - buildId = buildId, runningIncrement = -1 + buildId = buildId, runningIncrement = -1, debug = debug ) } } @@ -315,9 +314,9 @@ class PipelineStageService @Autowired constructor( pipelineBuildDao.updateBuildStageStatus( dslContext = context, projectId = projectId, buildId = buildId, stageStatus = allStageStatus ) - if (!debug) pipelineBuildSummaryDao.updateRunningCount( + pipelineBuildSummaryDao.updateRunningCount( dslContext = context, projectId = projectId, pipelineId = pipelineId, - buildId = buildId, runningIncrement = 1 + buildId = buildId, runningIncrement = 1, debug = debug ) } } @@ -428,9 +427,9 @@ class PipelineStageService @Autowired constructor( oldBuildStatus = BuildStatus.STAGE_SUCCESS, newBuildStatus = BuildStatus.RUNNING ) // #4255 stage审核超时恢复运行状态需要将运行状态+1，即使直接结束也会在finish阶段减回来 - if (!debug) pipelineBuildSummaryDao.updateRunningCount( + pipelineBuildSummaryDao.updateRunningCount( dslContext = context, projectId = projectId, pipelineId = pipelineId, - buildId = buildId, runningIncrement = 1 + buildId = buildId, runningIncrement = 1, debug = debug ) } } @@ -472,7 +471,7 @@ class PipelineStageService @Autowired constructor( ), position = ControlPointPosition.BEFORE_POSITION, stageId = stageId, - notifyType = NotifyUtils.checkNotifyType(checkIn?.notifyType) ?: mutableSetOf(), + notifyType = NotifyUtils.checkNotifyType(checkIn?.notifyType), markdownContent = checkIn?.markdownContent ) // #3400 FinishEvent会刷新HISTORY列表的Stage状态 @@ -567,6 +566,15 @@ class PipelineStageService @Autowired constructor( return pipelineBuildStageDao.getOneByStatus(dslContext, projectId, buildId, pendingStatusSet) } + fun getPendingStages(projectId: String, buildId: String): List { + return pipelineBuildStageDao.listBuildStages( + dslContext = dslContext, + projectId = projectId, + buildId = buildId, + statusSet = pendingStatusSet + ) + } + fun pauseStageNotify( userId: String, triggerUserId: String, @@ -608,7 +616,7 @@ class PipelineStageService @Autowired constructor( ), position = ControlPointPosition.BEFORE_POSITION, stageId = stage.stageId, - notifyType = NotifyUtils.checkNotifyType(checkIn.notifyType) ?: mutableSetOf(), + notifyType = NotifyUtils.checkNotifyType(checkIn.notifyType), markdownContent = checkIn.markdownContent ) ) diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/detail/BaseBuildDetailService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/detail/BaseBuildDetailService.kt index 12fcd088ab1..df7fcf208e0 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/detail/BaseBuildDetailService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/detail/BaseBuildDetailService.kt @@ -41,6 +41,7 @@ import com.tencent.devops.common.pipeline.Model import com.tencent.devops.common.pipeline.container.Container import com.tencent.devops.common.pipeline.container.Stage import com.tencent.devops.common.pipeline.enums.BuildStatus +import com.tencent.devops.common.pipeline.enums.ChannelCode import com.tencent.devops.common.pipeline.pojo.element.Element import com.tencent.devops.common.redis.RedisOperation import com.tencent.devops.common.service.utils.LogUtils @@ -139,11 +140,11 @@ open class BaseBuildDetailService constructor( return JsonUtil.to(record!!.model, Model::class.java) } finally { lock.unlock() -// logger.info("[$buildId|$buildStatus]|$operation|update_detail_model| $message") -// if (message == "update done") { // 防止MQ异常导致锁时间过长，将推送事件移出锁定范围 -// watcher.start("dispatchEvent") -// pipelineDetailChangeEvent(projectId, buildId) -// } + logger.info("[$buildId|$buildStatus]|$operation|update_detail_model| $message") + if (message == "update done") { // 防止MQ异常导致锁时间过长，将推送事件移出锁定范围 + watcher.start("dispatchEvent") + pipelineDetailChangeEvent(projectId, buildId) + } LogUtils.printCostTimeWE(watcher) } } @@ -243,10 +244,11 @@ open class BaseBuildDetailService constructor( } } - protected fun pipelineDetailChangeEvent(projectId: String, buildId: String) { - val pipelineBuildInfo = pipelineBuildDao.getBuildInfo(dslContext, projectId, buildId) ?: return - // 异步转发，解耦核心 - pipelineEventDispatcher.dispatch( + private fun pipelineDetailChangeEvent(projectId: String, buildId: String) { + val pipelineBuildInfo = pipelineBuildDao.getBuildInfo(dslContext, projectId, buildId) + if (pipelineBuildInfo?.channelCode == ChannelCode.GIT) pipelineEventDispatcher.dispatch( + // 异步转发，解耦核心 + // TODO stream内部和开源前端未更新前，保持推送 PipelineBuildWebSocketPushEvent( source = "recordDetail", projectId = pipelineBuildInfo.projectId, diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/record/BaseBuildRecordService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/record/BaseBuildRecordService.kt index 4c5a826eabb..4ea5f9fd5a5 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/record/BaseBuildRecordService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/record/BaseBuildRecordService.kt @@ -57,6 +57,7 @@ import com.tencent.devops.process.engine.dao.PipelineResourceDao import com.tencent.devops.process.engine.dao.PipelineResourceVersionDao import com.tencent.devops.process.engine.pojo.event.PipelineBuildWebSocketPushEvent import com.tencent.devops.process.engine.service.PipelineElementService +import com.tencent.devops.process.engine.utils.PipelineUtils import com.tencent.devops.process.pojo.BuildStageStatus import com.tencent.devops.process.pojo.pipeline.record.BuildRecordModel import com.tencent.devops.process.pojo.pipeline.record.BuildRecordStage @@ -189,6 +190,9 @@ open class BaseBuildRecordService( return try { watcher.start("fillElementWhenNewBuild") val fullModel = JsonUtil.to(resourceStr, Model::class.java) + fullModel.stages.forEach { + PipelineUtils.transformUserIllegalReviewParams(it.checkIn?.reviewParams) + } val baseModelMap = JsonUtil.toMutableMap(bean = fullModel, skipEmpty = false) val mergeBuildRecordParam = MergeBuildRecordParam( projectId = projectId, diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/record/ContainerBuildRecordService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/record/ContainerBuildRecordService.kt index f6697d9d436..c3cdec48b60 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/record/ContainerBuildRecordService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/record/ContainerBuildRecordService.kt @@ -249,9 +249,6 @@ class ContainerBuildRecordService( if (recordContainer.endTime == null) { endTime = LocalDateTime.now() } - if (!BuildStatus.parse(containerVar[Container::startVMStatus.name]?.toString()).isFinish()) { - containerVar[Container::startVMStatus.name] = buildStatus.name - } newTimestamps[BuildTimestampType.JOB_CONTAINER_SHUTDOWN] = BuildRecordTimeStamp( null, LocalDateTime.now().timestampmilli() ) diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/record/TaskBuildRecordService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/record/TaskBuildRecordService.kt index bb8d2513b58..92334b17a11 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/record/TaskBuildRecordService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/record/TaskBuildRecordService.kt @@ -522,6 +522,25 @@ class TaskBuildRecordService( ) } + fun updateAsyncStatus( + projectId: String, + pipelineId: String, + buildId: String, + taskId: String, + executeCount: Int, + asyncStatus: String + ) { + recordTaskDao.updateAsyncStatus( + dslContext = dslContext, + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + taskId = taskId, + executeCount = executeCount, + asyncStatus = asyncStatus + ) + } + companion object { private val logger = LoggerFactory.getLogger(TaskBuildRecordService::class.java) private const val TASK_PAUSE_TAG_VAR = "taskPause" diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/vmbuild/EngineVMBuildService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/vmbuild/EngineVMBuildService.kt index f755df9ee2a..95aa842e507 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/vmbuild/EngineVMBuildService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/service/vmbuild/EngineVMBuildService.kt @@ -113,7 +113,8 @@ import org.springframework.stereotype.Service "ReturnCount", "TooManyFunctions", "MagicNumber", - "LargeClass" + "LargeClass", + "ComplexMethod" ) @Service class EngineVMBuildService @Autowired(required = false) constructor( @@ -276,12 +277,14 @@ class EngineVMBuildService @Autowired(required = false) constructor( variablesWithType.addAll(customBuildParameters) Triple(envList, contextMap, tm) } + is NormalContainer -> { val tm = transMinuteTimeoutToMills(container.controlOption.jobControlOption.timeout) val contextMap = pipelineContextService.getAllBuildContext(variables).toMutableMap() fillContainerContext(contextMap, null, c.matrixContext, asCodeSettings?.enable) Triple(mutableListOf(), contextMap, tm) } + else -> throw OperationException("vmName($vmName) is an illegal container type: $c") } buildingHeartBeatUtils.addHeartBeat(buildId, vmSeqId, System.currentTimeMillis()) @@ -409,14 +412,17 @@ class EngineVMBuildService @Autowired(required = false) constructor( message = "Job Timeout" ActionType.TERMINATE } + finalBuildStatus.isFailure() -> { message = "Agent failed to start!" ActionType.TERMINATE } + finalBuildStatus.isCancel() -> { message = "Job Cancel" ActionType.TERMINATE } + else -> { message = "Agent startup!" ActionType.START @@ -527,19 +533,23 @@ class EngineVMBuildService @Autowired(required = false) constructor( LOG.info("ENGINE|$buildId|BC_QUEUE|${task.projectId}|j($vmSeqId)|${task.taskId}|${task.taskName}") BuildTask(buildId, vmSeqId, BuildTaskStatus.WAIT, task.executeCount) } + task.taskAtom.isNotBlank() -> { // 排除非构建机的插件任务 继续等待直到它完成 LOG.info("ENGINE|$buildId|BC_NOT_VM|${task.projectId}|j($vmSeqId)|${task.taskId}|${task.taskName}") BuildTask(buildId, vmSeqId, BuildTaskStatus.WAIT, task.executeCount) } + task.taskId == VMUtils.genEndPointTaskId(task.taskSeq) -> { // 全部完成了 pipelineRuntimeService.claimBuildTask(task, userId) // 刷新一下这个结束的任务节点时间 BuildTask(buildId, vmSeqId, BuildTaskStatus.END, task.executeCount) } + pipelineTaskService.isNeedPause(taskId = task.taskId, buildId = task.buildId, taskRecord = task) -> { // 如果插件配置了前置暂停, 暂停期间关闭当前构建机，节约资源。 pipelineTaskService.executePause(taskId = task.taskId, buildId = task.buildId, taskRecord = task) LOG.info("ENGINE|$buildId|BC_PAUSE|${task.projectId}|j($vmSeqId)|${task.taskId}|${task.taskAtom}") pipelineEventDispatcher.dispatch( + // task 前置暂停 PipelineBuildStatusBroadCastEvent( source = "TaskPause-${task.containerId}-${task.buildId}", projectId = task.projectId, @@ -548,7 +558,13 @@ class EngineVMBuildService @Autowired(required = false) constructor( buildId = task.buildId, taskId = task.taskId, stageId = task.stageId, - actionType = ActionType.REFRESH + containerHashId = task.containerHashId, + jobId = task.jobId, + stepId = task.stepId, + atomCode = task.atomCode, + executeCount = task.executeCount, + actionType = ActionType.REFRESH, + buildStatus = task.status.name ) ) BuildTask(buildId, vmSeqId, BuildTaskStatus.END, task.executeCount) @@ -569,6 +585,7 @@ class EngineVMBuildService @Autowired(required = false) constructor( ) BuildTask(buildId, vmSeqId, BuildTaskStatus.WAIT, task.executeCount) } + else -> { val allVariable = buildVariableService.getAllVariable(task.projectId, task.pipelineId, buildId) // 构造扩展变量 @@ -595,9 +612,13 @@ class EngineVMBuildService @Autowired(required = false) constructor( jmxElements.execute(task.taskType) } pipelineEventDispatcher.dispatch( + // market task 启动 PipelineBuildStatusBroadCastEvent( source = "vm-build-claim($vmSeqId)", projectId = task.projectId, pipelineId = task.pipelineId, - userId = task.starter, buildId = buildId, taskId = task.taskId, actionType = ActionType.START + userId = task.starter, buildId = buildId, taskId = task.taskId, actionType = ActionType.START, + containerHashId = task.containerHashId, jobId = task.jobId, stageId = task.stageId, + stepId = task.stepId, atomCode = task.atomCode, executeCount = task.executeCount, + buildStatus = BuildStatus.RUNNING.name ) ) val signToken = UUIDUtil.generate() @@ -783,6 +804,18 @@ class EngineVMBuildService @Autowired(required = false) constructor( source = "completeClaimBuildTask", sendEventFlag = false ) + + pipelineEventDispatcher.dispatch( + // market task 结束 + PipelineBuildStatusBroadCastEvent( + source = "task-end-${result.taskId}", projectId = buildInfo.projectId, + pipelineId = buildInfo.pipelineId, userId = buildInfo.startUser, + buildId = buildId, taskId = result.taskId, actionType = ActionType.END, + containerHashId = task.containerHashId, jobId = task.jobId, stageId = task.stageId, + stepId = task.stepId, atomCode = task.atomCode, executeCount = task.executeCount, + buildStatus = task.status.name + ) + ) } if (buildStatus.isFailure()) { @@ -808,13 +841,6 @@ class EngineVMBuildService @Autowired(required = false) constructor( } } - pipelineEventDispatcher.dispatch( - PipelineBuildStatusBroadCastEvent( - source = "task-end-${result.taskId}", projectId = buildInfo.projectId, - pipelineId = buildInfo.pipelineId, userId = buildInfo.startUser, - buildId = buildId, taskId = result.taskId, actionType = ActionType.END - ) - ) // 发送度量数据 sendElementData(projectId = projectId, buildId = buildId, result = result) @@ -858,6 +884,7 @@ class EngineVMBuildService @Autowired(required = false) constructor( LOG.warn("ENGINE|$buildId|BCT_CANCEL_NOT_FINISH|${buildInfo.projectId}|job#$vmSeqId|$taskId") BuildStatus.CANCELED } + result.success -> { pipelineTaskService.removeRetryCache(buildId, result.taskId) pipelineTaskService.removeFailTaskVar( @@ -866,6 +893,7 @@ class EngineVMBuildService @Autowired(required = false) constructor( ) // 清理插件错误信息（重试插件成功的情况下） BuildStatus.SUCCEED } + else -> { when { pipelineTaskService.isRetryWhenFail( @@ -884,6 +912,7 @@ class EngineVMBuildService @Autowired(required = false) constructor( ) BuildStatus.RETRY } + else -> { // 记录错误插件信息 pipelineTaskService.createFailTaskVar( buildId = buildId, projectId = buildInfo.projectId, diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/utils/PipelineUtils.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/utils/PipelineUtils.kt index 270160bd9b0..062c79a5358 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/utils/PipelineUtils.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/engine/utils/PipelineUtils.kt @@ -118,6 +118,15 @@ object PipelineUtils { } } + fun transformUserIllegalReviewParams(reviewParams: List?) { + reviewParams?.forEach { param -> + val value = param.value + if (param.valueType == ManualReviewParamType.MULTIPLE && value is String && value.isBlank()) { + param.value = null + } + } + } + private fun checkVariablesLength(key: String, value: String) { if (value.length >= PIPELINE_VARIABLES_STRING_LENGTH_MAX) { throw ErrorCodeException( @@ -149,6 +158,39 @@ object PipelineUtils { return stages } + /** + * 将流水线常量转换成模板常量 + */ + fun fixedTemplateParam(model: Model): Model { + val triggerContainer = model.stages[0].containers[0] as TriggerContainer + val params = mutableListOf() + val templateParams = mutableListOf() + triggerContainer.params.forEach { + if (it.constant == true) { + templateParams.add(it) + } else { + params.add(it) + } + } + val fixedTriggerContainer = triggerContainer.copy( + params = params, + templateParams = if (templateParams.isEmpty()) { + null + } else { + templateParams + } + ) + val stages = ArrayList() + model.stages.forEachIndexed { index, stage -> + if (index == 0) { + stages.add(stage.copy(containers = listOf(fixedTriggerContainer))) + } else { + stages.add(stage) + } + } + return model.copy(stages = stages) + } + /** * 通过流水线参数和模板编排生成新Model */ @@ -169,7 +211,9 @@ object PipelineUtils { BuildPropertyCompatibilityTools.mergeProperties( from = templateTrigger.params, to = BuildPropertyCompatibilityTools.mergeProperties( - from = templateTrigger.templateParams!!, to = param ?: emptyList() + // 模板常量需要变成流水线常量 + from = templateTrigger.templateParams!!.map { it.copy(constant = true) }, + to = param ?: emptyList() ) ) } diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/plugin/ElementBizPlugin.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/plugin/ElementBizPlugin.kt index 06b79cd8713..42ec65721a6 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/plugin/ElementBizPlugin.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/plugin/ElementBizPlugin.kt @@ -33,6 +33,7 @@ import com.tencent.devops.common.pipeline.enums.ChannelCode import com.tencent.devops.common.pipeline.pojo.element.Element import com.tencent.devops.common.pipeline.pojo.element.atom.BeforeDeleteParam import com.tencent.devops.common.pipeline.pojo.element.atom.ElementCheckResult +import com.tencent.devops.process.pojo.pipeline.PipelineYamlVo /** * 流水线的Element的编排插件处理器 @@ -58,7 +59,8 @@ interface ElementBizPlugin { userId: String, channelCode: ChannelCode = ChannelCode.BS, create: Boolean, - container: Container + container: Container, + yamlInfo: PipelineYamlVo? ) /** diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/BuildVariableService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/BuildVariableService.kt index 5685895238d..86328573d6e 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/BuildVariableService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/BuildVariableService.kt @@ -40,6 +40,7 @@ import com.tencent.devops.process.utils.PipelineVarUtil import org.apache.commons.lang3.math.NumberUtils import org.jooq.DSLContext import org.jooq.impl.DSL +import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service @@ -52,6 +53,10 @@ class BuildVariableService @Autowired constructor( private val redisOperation: RedisOperation ) { + companion object { + private val logger = LoggerFactory.getLogger(BuildVariableService::class.java) + } + /** * 获取构建执行次数（重试次数+1），如没有重试过，则为1 */ @@ -277,13 +282,15 @@ class BuildVariableService @Autowired constructor( // 加锁防止数据被重复插入 redisLock.lock() watch.start("getVars") - val buildVarMap = pipelineBuildVarDao.getVars(dslContext, projectId, buildId) + val buildVarMap = pipelineBuildVarDao.getBuildVarMap(dslContext, projectId, buildId) val insertBuildParameters = mutableListOf() val updateBuildParameters = mutableListOf() pipelineBuildParameters.forEach { if (!buildVarMap.containsKey(it.key)) { insertBuildParameters.add(it) } else { + // TODO PAC上线后删除, 打印流水线运行时覆盖只读变量,只在第一次运行时输出,重试不输出 + printReadOnlyVar(buildVarMap, variables, it, projectId, pipelineId, buildId) updateBuildParameters.add(it) } } @@ -308,6 +315,25 @@ class BuildVariableService @Autowired constructor( } } + private fun printReadOnlyVar( + buildVarMap: Map, + variables: Map, + buildParameters: BuildParameters, + projectId: String, + pipelineId: String, + buildId: String + ) { + if (buildVarMap[PIPELINE_RETRY_COUNT] == null && + variables[PIPELINE_RETRY_COUNT] == null && + buildVarMap[buildParameters.key]?.readOnly == true + ) { + logger.warn( + "BKSystemErrorMonitor|$projectId|$pipelineId|$buildId|${buildParameters.key}| " + + "build var read-only cannot be modified" + ) + } + } + // #10082 查询Agent复用互斥使用的AgentId fun fetchAgentReuseMutexVar( projectId: String, diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/SubPipelineStatusService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/SubPipelineStatusService.kt index 261ec311afa..72266fc0763 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/SubPipelineStatusService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/SubPipelineStatusService.kt @@ -30,21 +30,33 @@ package com.tencent.devops.process.service import com.tencent.devops.common.api.pojo.ErrorCode import com.tencent.devops.common.api.pojo.ErrorType import com.tencent.devops.common.api.util.JsonUtil +import com.tencent.devops.common.event.dispatcher.pipeline.PipelineEventDispatcher import com.tencent.devops.common.event.pojo.pipeline.PipelineBuildFinishBroadCastEvent +import com.tencent.devops.common.event.pojo.pipeline.PipelineBuildQueueBroadCastEvent import com.tencent.devops.common.pipeline.enums.BuildStatus import com.tencent.devops.common.pipeline.enums.StartType import com.tencent.devops.common.redis.RedisOperation import com.tencent.devops.common.web.utils.I18nUtil +import com.tencent.devops.common.websocket.enum.RefreshType import com.tencent.devops.process.constant.ProcessMessageCode.ERROR_NO_BUILD_RECORD_FOR_CORRESPONDING_SUB_PIPELINE +import com.tencent.devops.process.engine.pojo.event.PipelineBuildWebSocketPushEvent import com.tencent.devops.process.engine.service.PipelineRuntimeService import com.tencent.devops.process.pojo.pipeline.SubPipelineStatus +import com.tencent.devops.process.utils.PIPELINE_START_SUB_RUN_MODE +import com.tencent.devops.process.utils.PIPELINE_START_PARENT_BUILD_ID +import com.tencent.devops.process.utils.PIPELINE_START_PARENT_BUILD_TASK_ID +import com.tencent.devops.process.utils.PIPELINE_START_PARENT_EXECUTE_COUNT +import com.tencent.devops.process.utils.PIPELINE_START_PARENT_PIPELINE_ID +import com.tencent.devops.process.utils.PIPELINE_START_PARENT_PROJECT_ID +import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service @Service class SubPipelineStatusService @Autowired constructor( private val pipelineRuntimeService: PipelineRuntimeService, - private val redisOperation: RedisOperation + private val redisOperation: RedisOperation, + private val pipelineEventDispatcher: PipelineEventDispatcher ) { companion object { @@ -52,6 +64,9 @@ class SubPipelineStatusService @Autowired constructor( private const val SUBPIPELINE_STATUS_START_EXPIRED = 54000L // 子流水线完成过期时间10分钟 private const val SUBPIPELINE_STATUS_FINISH_EXPIRED = 600L + private val logger = LoggerFactory.getLogger(SubPipelineStatusService::class.java) + // 异步启动子流水线 + private const val ASYNC_RUN_MODE = "asyn" } fun onStart(buildId: String) { @@ -64,24 +79,153 @@ class SubPipelineStatusService @Autowired constructor( ) } + /** + * 异步启动子流水线 + */ + fun onBuildQueue(event: PipelineBuildQueueBroadCastEvent) { + with(event) { + // 不是流水线启动 + if (triggerType != StartType.PIPELINE.name) { + return + } + try { + updateParentPipelineTaskStatus( + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + asyncStatus = BuildStatus.RUNNING.name + ) + } catch (ignored: Exception) { + logger.warn("fail to update parent pipeline task status", ignored) + } + } + } + fun onFinish(event: PipelineBuildFinishBroadCastEvent) { with(event) { - // 不是子流水线启动或者子流水线是异步启动的，不需要缓存状态 - if (triggerType != StartType.PIPELINE.name || - redisOperation.get(getSubPipelineStatusKey(buildId)) == null - ) { + // 不是流水线启动 + if (triggerType != StartType.PIPELINE.name) { return } + try { + val (buildSourceStatus, pipelineStatus) = getSubPipelineStatusFromDB(projectId, buildId) + updateParentPipelineTaskStatus( + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + asyncStatus = when { + buildSourceStatus.isSuccess() -> BuildStatus.SUCCEED + buildSourceStatus.isCancel() -> BuildStatus.CANCELED + else -> BuildStatus.FAILED + }.name + ) + // 子流水线是异步启动的，不需要缓存状态 + if (redisOperation.get(getSubPipelineStatusKey(buildId)) != null) { + redisOperation.set( + key = getSubPipelineStatusKey(buildId), + value = JsonUtil.toJson(pipelineStatus), + expiredInSecond = SUBPIPELINE_STATUS_FINISH_EXPIRED + ) + } + } catch (ignored: Exception) { + logger.warn("fail to update parent pipeline task status", ignored) + } + } + } - redisOperation.set( - key = getSubPipelineStatusKey(buildId), - value = JsonUtil.toJson(getSubPipelineStatusFromDB(event.projectId, buildId)), - expiredInSecond = SUBPIPELINE_STATUS_FINISH_EXPIRED + /** + * 更新父流水线插件异步执行状态 + */ + private fun updateParentPipelineTaskStatus( + projectId: String, + pipelineId: String, + buildId: String, + asyncStatus: String + ) { + // 父流水线相关信息 + val keys = setOf( + PIPELINE_START_PARENT_PROJECT_ID, + PIPELINE_START_PARENT_PIPELINE_ID, + PIPELINE_START_PARENT_BUILD_ID, + PIPELINE_START_PARENT_BUILD_TASK_ID, + PIPELINE_START_SUB_RUN_MODE, + PIPELINE_START_PARENT_EXECUTE_COUNT + ) + val buildVariables = pipelineRuntimeService.getBuildVariableService( + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + keys = keys + ) + keys.forEach { + // 存在异常值，则直接返回 + if (buildVariables[it].isNullOrBlank()) { + logger.warn( + "The parent pipeline status cannot be updated, " + + "because an abnormal variable exists[$buildVariables]" + ) + return + } + } + // 非异步触发不处理 + if (buildVariables[PIPELINE_START_SUB_RUN_MODE] != ASYNC_RUN_MODE) { + return + } + pipelineRuntimeService.getBuildInfo( + projectId = buildVariables[PIPELINE_START_PARENT_PROJECT_ID]!!, + pipelineId = buildVariables[PIPELINE_START_PARENT_PIPELINE_ID]!!, + buildId = buildVariables[PIPELINE_START_PARENT_BUILD_ID]!! + )?.let { + updatePipelineTaskStatus( + projectId = it.projectId, + pipelineId = it.pipelineId, + buildId = it.buildId, + taskId = buildVariables[PIPELINE_START_PARENT_BUILD_TASK_ID]!!, + executeCount = buildVariables[PIPELINE_START_PARENT_EXECUTE_COUNT]!!.toInt(), + userId = it.startUser, + asyncStatus = asyncStatus ) } } - private fun getSubPipelineStatusFromDB(projectId: String, buildId: String): SubPipelineStatus { + /** + * 更新当前流水线插件异步执行状态 + */ + fun updatePipelineTaskStatus( + projectId: String, + pipelineId: String, + buildId: String, + taskId: String, + executeCount: Int, + userId: String, + asyncStatus: String + ) { + logger.info( + "start update parent pipeline asyncStatus[$asyncStatus]|$projectId|$pipelineId|" + + "$buildId|$executeCount" + ) + pipelineRuntimeService.updateAsyncStatus( + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId, + taskId = taskId, + executeCount = executeCount, + asyncStatus = asyncStatus + ) + pipelineEventDispatcher.dispatch( + PipelineBuildWebSocketPushEvent( + source = "updateTaskStatus", + projectId = projectId, + pipelineId = pipelineId, + userId = userId, + buildId = buildId, + refreshTypes = RefreshType.RECORD.binary, + executeCount = executeCount + ) + ) + } + + private fun getSubPipelineStatusFromDB(projectId: String, buildId: String): Pair { val buildInfo = pipelineRuntimeService.getBuildInfo(projectId, buildId) return if (buildInfo != null) { val status: BuildStatus = when { @@ -91,11 +235,11 @@ class SubPipelineStatusService @Autowired constructor( buildInfo.isStageSuccess() -> BuildStatus.RUNNING // stage 特性， 未结束，只是卡在Stage审核中 else -> buildInfo.status } - SubPipelineStatus( + buildInfo.status to SubPipelineStatus( status = status.name ) } else { - SubPipelineStatus( + BuildStatus.FAILED to SubPipelineStatus( status = BuildStatus.FAILED.name, errorType = ErrorType.USER, errorCode = ErrorCode.USER_RESOURCE_NOT_FOUND, @@ -107,7 +251,7 @@ class SubPipelineStatusService @Autowired constructor( fun getSubPipelineStatus(projectId: String, buildId: String): SubPipelineStatus { val subPipelineStatusStr = redisOperation.get(getSubPipelineStatusKey(buildId)) return if (subPipelineStatusStr.isNullOrBlank()) { - getSubPipelineStatusFromDB(projectId, buildId) + getSubPipelineStatusFromDB(projectId, buildId).second } else { val redisSubPipelineStatus = JsonUtil.to(subPipelineStatusStr, SubPipelineStatus::class.java) // 如果状态完成,子流水线插件就不会再调用,从redis中删除key diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineBuildService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineBuildService.kt index df2846ecf99..39a97b229f9 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineBuildService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineBuildService.kt @@ -31,10 +31,12 @@ import com.tencent.bk.audit.annotations.ActionAuditRecord import com.tencent.bk.audit.annotations.AuditAttribute import com.tencent.bk.audit.annotations.AuditInstanceRecord import com.tencent.devops.common.api.exception.ErrorCodeException +import com.tencent.devops.common.api.util.JsonUtil import com.tencent.devops.common.audit.ActionAuditContent import com.tencent.devops.common.auth.api.ActionId import com.tencent.devops.common.auth.api.ResourceTypeId import com.tencent.devops.common.pipeline.Model +import com.tencent.devops.common.pipeline.container.TriggerContainer import com.tencent.devops.common.pipeline.enums.BuildFormPropertyType import com.tencent.devops.common.pipeline.enums.ChannelCode import com.tencent.devops.common.pipeline.enums.StartType @@ -177,7 +179,7 @@ class PipelineBuildService( // 如果指定了版本号，则设置指定的版本号 pipeline.version = signPipelineVersion ?: pipeline.version - + val originModelStr = JsonUtil.toJson(model, formatted = false) // 只有新构建才需要填充Post插件与质量红线插件 if (!pipelineParamMap.containsKey(PIPELINE_RETRY_BUILD_ID)) { pipelineElementService.fillElementWhenNewBuild( @@ -209,12 +211,14 @@ class PipelineBuildService( pipelineId = pipeline.pipelineId, buildId = buildId, resourceVersion = pipeline.version, - model = model, + modelStr = originModelStr, pipelineSetting = setting, currentBuildNo = buildNo, triggerReviewers = triggerReviewers, pipelineParamMap = pipelineParamMap, webHookStartParam = webHookStartParam, + // 解析出定义的流水线变量 + realStartParamKeys = (model.stages[0].containers[0] as TriggerContainer).params.map { it.id }, debug = debug ?: false, versionName = versionName, yamlVersion = yamlVersion diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineSettingVersionService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineSettingVersionService.kt index 5e79ea7b995..cf1ce8255cb 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineSettingVersionService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineSettingVersionService.kt @@ -115,7 +115,7 @@ class PipelineSettingVersionService @Autowired constructor( settingInfo.failSubscriptionList = ve.failSubscriptionList ?: settingInfo.failSubscriptionList // 这里不应该出现错误的流水线名，但保留历史留下的处理方式 settingInfo.pipelineName = ve.pipelineName ?: settingInfo.pipelineName - settingInfo.labels = ve.labels ?: listOf() + settingInfo.labels = ve.labels ?: labels settingInfo.desc = ve.desc ?: settingInfo.desc settingInfo.buildNumRule = ve.buildNumRule ?: settingInfo.buildNumRule // #8161 如果是PAC发布前产生的数据，则流水线名称为空，可以用正式配置覆盖 diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineStatusService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineStatusService.kt index 31c81702b55..144d49120c5 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineStatusService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineStatusService.kt @@ -31,13 +31,11 @@ import com.tencent.devops.common.api.util.timestampmilli import com.tencent.devops.common.pipeline.enums.BuildStatus import com.tencent.devops.common.pipeline.enums.ChannelCode import com.tencent.devops.common.pipeline.utils.BuildStatusSwitcher -import com.tencent.devops.process.dao.PipelineSettingDao import com.tencent.devops.process.engine.dao.PipelineBuildDao import com.tencent.devops.process.engine.dao.PipelineBuildTaskDao import com.tencent.devops.process.engine.dao.PipelineInfoDao import com.tencent.devops.process.engine.service.PipelineRuntimeService import com.tencent.devops.process.pojo.PipelineStatus -import com.tencent.devops.common.pipeline.pojo.setting.PipelineRunLockType import org.jooq.DSLContext import org.springframework.stereotype.Service @@ -46,7 +44,6 @@ import org.springframework.stereotype.Service class PipelineStatusService( private val dslContext: DSLContext, private val pipelineInfoDao: PipelineInfoDao, - private val pipelineSettingDao: PipelineSettingDao, private val pipelineBuildTaskDao: PipelineBuildTaskDao, private val pipelineBuildDao: PipelineBuildDao, private val pipelineRuntimeService: PipelineRuntimeService @@ -60,7 +57,6 @@ class PipelineStatusService( channelCode = ChannelCode.BS, pipelineId = pipelineId ) ?: return null - val pipelineSetting = pipelineSettingDao.getSetting(dslContext, projectId, pipelineId) ?: return null val pipelineBuildSummary = pipelineRuntimeService.getBuildSummaryRecord(projectId, pipelineId) ?: return null val buildStatusOrd = pipelineBuildSummary.latestStatus val finishCount = pipelineBuildSummary.finishCount ?: 0 @@ -95,7 +91,7 @@ class PipelineStatusService( latestBuildStartTime = (pipelineBuildSummary.latestStartTime)?.timestampmilli() ?: 0, latestBuildStatus = pipelineBuildStatus, // latestBuildTaskName = pipelineBuildSummary.latestTaskName, - lock = PipelineRunLockType.checkLock(pipelineSetting.runLockType.ordinal), + lock = pipelineInfo.locked, runningBuildCount = pipelineBuildSummary.runningCount ?: 0, lastBuildFinishCount = lastBuildFinishCount, lastBuildTotalCount = lastBuildTotalCount, diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineTransferYamlService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineTransferYamlService.kt index ddc8a297494..64bb3aee663 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineTransferYamlService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineTransferYamlService.kt @@ -141,7 +141,6 @@ class PipelineTransferYamlService @Autowired constructor( aspectWrapper = PipelineTransferAspectWrapper(aspects) ) ) - val newYaml = TransferMapper.mergeYaml(data.oldYaml, TransferMapper.toYaml(response)) if (invalidElement.isNotEmpty()) { throw PipelineTransferException( ELEMENT_NOT_SUPPORT_TRANSFER, @@ -149,6 +148,7 @@ class PipelineTransferYamlService @Autowired constructor( ) } watcher.start("step_2|mergeYaml") + val newYaml = TransferMapper.mergeYaml(data.oldYaml, TransferMapper.toYaml(response)) watcher.stop() logger.info(watcher.toString()) return TransferResponse( diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/record/PipelineRecordModelService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/record/PipelineRecordModelService.kt index dc87346d51a..3a20008e2e3 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/record/PipelineRecordModelService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/record/PipelineRecordModelService.kt @@ -270,15 +270,10 @@ class PipelineRecordModelService @Autowired constructor( // 获取开机任务的序号 val startVMTaskSeq = buildRecordContainer.containerVar[Container::startVMTaskSeq.name]?.toString()?.toInt() ?: 1 containerRecordTasks.forEach { containerRecordTask -> - if (startVMTaskSeq > 1 && startVMTaskSeq > containerRecordTask.taskSeq) { - // 当开机任务的序号大于1时，说明第一个任务不是开机任务，job含有内置插件任务，需要重新调整开机任务前面的task任务的taskSeq值 - containerRecordTask.taskSeq += 1 - } + handleTaskSeq(startVMTaskSeq, containerRecordTask) val taskVarMap = generateTaskVarMap( - containerRecordTask = containerRecordTask, - taskId = containerRecordTask.taskId, - containerBaseMap = containerBaseMap, - matrixTaskFlag = matrixTaskFlag, + containerRecordTask = containerRecordTask, taskId = containerRecordTask.taskId, + containerBaseMap = containerBaseMap, matrixTaskFlag = matrixTaskFlag, taskBaseMaps = taskBaseMaps ) while (containerRecordTask.taskSeq - preContainerRecordTaskSeq > 1) { @@ -319,6 +314,16 @@ class PipelineRecordModelService @Autowired constructor( } } + private fun handleTaskSeq( + startVMTaskSeq: Int, + containerRecordTask: BuildRecordTask + ) { + if (startVMTaskSeq < 1 || (startVMTaskSeq > 1 && startVMTaskSeq > containerRecordTask.taskSeq)) { + // 当开机任务的序号大于1时，说明第一个任务不是开机任务，job含有内置插件任务，需要重新调整开机任务前面的task任务的taskSeq值 + containerRecordTask.taskSeq += 1 + } + } + private fun generateTaskVarMap( containerRecordTask: BuildRecordTask, taskId: String, @@ -330,6 +335,7 @@ class PipelineRecordModelService @Autowired constructor( taskVarMap[Element::id.name] = taskId taskVarMap[Element::status.name] = containerRecordTask.status ?: "" taskVarMap[Element::executeCount.name] = containerRecordTask.executeCount + taskVarMap[Element::asyncStatus.name] = containerRecordTask.asyncStatus ?: "" val elementPostInfo = containerRecordTask.elementPostInfo if (elementPostInfo != null) { // 生成post类型task的变量模型 @@ -442,7 +448,7 @@ class PipelineRecordModelService @Autowired constructor( // 如果跳过的是矩阵类task，则需要生成完整的model对象以便合并 taskVarMap["@type"] = MatrixStatusElement.classType taskVarMap[MatrixStatusElement::originClassType.name] = - taskBaseMap[MatrixStatusElement::classType.name].toString() + taskBaseMap[MatrixStatusElement.classType].toString() taskVarMap[MatrixStatusElement::originAtomCode.name] = taskBaseMap[KEY_ATOM_CODE].toString() taskVarMap[MatrixStatusElement::originTaskAtom.name] = taskBaseMap[KEY_TASK_ATOM].toString() taskVarMap = ModelUtils.generateBuildModelDetail(taskBaseMap.deepCopy(), taskVarMap) diff --git a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/scm/ScmProxyService.kt b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/scm/ScmProxyService.kt index e4b26d4305e..ccb44886750 100644 --- a/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/scm/ScmProxyService.kt +++ b/src/backend/ci/core/process/biz-base/src/main/kotlin/com/tencent/devops/process/service/scm/ScmProxyService.kt @@ -69,6 +69,9 @@ import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service import java.net.URLEncoder +import java.time.LocalDateTime +import java.time.ZoneId +import java.time.format.DateTimeFormatter import java.util.Base64 import javax.ws.rs.NotFoundException @@ -249,7 +252,7 @@ class ScmProxyService @Autowired constructor(private val client: Client) { } else -> { - throw IllegalArgumentException("Unknown repo($repo)") + null } } } @@ -289,7 +292,9 @@ class ScmProxyService @Autowired constructor(private val client: Client) { token = credInfo.first, region = null, userName = repo.userName, - search = search + search = search, + page = 1, + pageSize = 100 ) } else { val credInfo = getCredential( @@ -306,7 +311,9 @@ class ScmProxyService @Autowired constructor(private val client: Client) { token = credInfo.privateKey, region = null, userName = credInfo.username, - search = search + search = search, + page = 1, + pageSize = 100 ) } } @@ -342,7 +349,9 @@ class ScmProxyService @Autowired constructor(private val client: Client) { token = getTGitAccessToken(repo.userName), region = null, userName = repo.userName, - search = search + search = search, + page = 1, + pageSize = 100 ) } else { val credInfo = getCredential( @@ -359,7 +368,9 @@ class ScmProxyService @Autowired constructor(private val client: Client) { token = credInfo.privateKey, region = null, userName = credInfo.username, - search = search + search = search, + page = 1, + pageSize = 100 ) } } @@ -466,7 +477,7 @@ class ScmProxyService @Autowired constructor(private val client: Client) { checkRepoID(repositoryConfig) val repo = getRepo(projectId, repositoryConfig) as? CodeGitRepository ?: throw ErrorCodeException(errorCode = ProcessMessageCode.GIT_INVALID) - val isOauth = repo.credentialId.isEmpty() + val isOauth = repo.authType == RepoAuthType.OAUTH val token = if (isOauth) { getAccessToken(repo.userName).first } else { @@ -693,9 +704,9 @@ class ScmProxyService @Autowired constructor(private val client: Client) { detailUrl: String, externalId: String, status: String, - startedAt: String?, + startedAt: LocalDateTime?, conclusion: String?, - completedAt: String? + completedAt: LocalDateTime? ) { logger.info("Project($projectId) update github commit($commitId) check runs") @@ -709,9 +720,9 @@ class ScmProxyService @Autowired constructor(private val client: Client) { detailsUrl = detailUrl, externalId = externalId, status = status, - startedAt = startedAt, + startedAt = startedAt?.atZone(ZoneId.systemDefault())?.format(DateTimeFormatter.ISO_INSTANT), conclusion = conclusion, - completedAt = completedAt + completedAt = completedAt?.atZone(ZoneId.systemDefault())?.format(DateTimeFormatter.ISO_INSTANT) ) client.get(ServiceGithubResource::class).updateCheckRuns( diff --git a/src/backend/ci/core/process/biz-base/src/test/kotlin/com/tencent/devops/process/TestBase.kt b/src/backend/ci/core/process/biz-base/src/test/kotlin/com/tencent/devops/process/TestBase.kt index 8cc552ce599..4fff14dca97 100644 --- a/src/backend/ci/core/process/biz-base/src/test/kotlin/com/tencent/devops/process/TestBase.kt +++ b/src/backend/ci/core/process/biz-base/src/test/kotlin/com/tencent/devops/process/TestBase.kt @@ -321,6 +321,7 @@ open class TestBase : BkCiAbstractTest() { stageId = stageId, containerId = id?.toString() ?: firstContainerId, containerHashId = containerHashId, + jobId = "job-123", seq = id ?: firstContainerIdInt, containerType = vmContainerType, status = status, diff --git a/src/backend/ci/core/process/biz-base/src/test/kotlin/com/tencent/devops/process/engine/compatibility/v2/V2BuildParametersCompatibilityTransformerTest.kt b/src/backend/ci/core/process/biz-base/src/test/kotlin/com/tencent/devops/process/engine/compatibility/v2/V2BuildParametersCompatibilityTransformerTest.kt index 74bf5c88d79..0bd399815f2 100644 --- a/src/backend/ci/core/process/biz-base/src/test/kotlin/com/tencent/devops/process/engine/compatibility/v2/V2BuildParametersCompatibilityTransformerTest.kt +++ b/src/backend/ci/core/process/biz-base/src/test/kotlin/com/tencent/devops/process/engine/compatibility/v2/V2BuildParametersCompatibilityTransformerTest.kt @@ -80,6 +80,7 @@ class V2BuildParametersCompatibilityTransformerTest { "illegalStartParam" to "i will be delete after call it" ) val startBuildParameter = buildParametersCompatibilityTransformer.parseTriggerParam( + userId = "", projectId = "", pipelineId = "", paramProperties = paramProperties, paramValues = paramValues ) assertEquals(2, startBuildParameter.size) diff --git a/src/backend/ci/core/process/biz-base/src/test/kotlin/com/tencent/devops/process/engine/context/StartBuildContextTest.kt b/src/backend/ci/core/process/biz-base/src/test/kotlin/com/tencent/devops/process/engine/context/StartBuildContextTest.kt index 42783175c82..51dbc69464e 100644 --- a/src/backend/ci/core/process/biz-base/src/test/kotlin/com/tencent/devops/process/engine/context/StartBuildContextTest.kt +++ b/src/backend/ci/core/process/biz-base/src/test/kotlin/com/tencent/devops/process/engine/context/StartBuildContextTest.kt @@ -27,7 +27,6 @@ package com.tencent.devops.process.engine.context -import com.tencent.devops.common.pipeline.Model import com.tencent.devops.common.pipeline.enums.BuildStatus import com.tencent.devops.common.pipeline.enums.ChannelCode import com.tencent.devops.common.pipeline.enums.StartType @@ -118,11 +117,8 @@ class StartBuildContextTest : TestBase() { buildId = buildId, resourceVersion = version, versionName = null, - model = Model( - name = "pipelineName", - desc = null, - stages = genStages(2, jobSize = 2, elementSize = 2, needFinally = false) - ), + realStartParamKeys = emptyList(), + modelStr = "", pipelineParamMap = pipelineParamMap, currentBuildNo = null, triggerReviewers = null, diff --git a/src/backend/ci/core/process/biz-base/src/test/kotlin/com/tencent/devops/process/engine/control/ControlUtilsTest.kt b/src/backend/ci/core/process/biz-base/src/test/kotlin/com/tencent/devops/process/engine/control/ControlUtilsTest.kt index 300a0c31957..ba612cdb158 100644 --- a/src/backend/ci/core/process/biz-base/src/test/kotlin/com/tencent/devops/process/engine/control/ControlUtilsTest.kt +++ b/src/backend/ci/core/process/biz-base/src/test/kotlin/com/tencent/devops/process/engine/control/ControlUtilsTest.kt @@ -27,7 +27,6 @@ package com.tencent.devops.process.engine.control -import com.tencent.devops.common.expression.ExpressionParseException import com.tencent.devops.common.pipeline.NameAndValue import com.tencent.devops.common.pipeline.enums.BuildStatus import com.tencent.devops.common.pipeline.enums.JobRunCondition @@ -45,7 +44,6 @@ import io.mockk.mockkObject import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test -import org.junit.jupiter.api.assertThrows /** * @version 1.0 @@ -116,8 +114,7 @@ class ControlUtilsTest : TestBase() { conditions = conditions, variables = variables, buildId = buildId, - runCondition = JobRunCondition.CUSTOM_VARIABLE_MATCH_NOT_RUN, - asCodeEnabled = true + runCondition = JobRunCondition.CUSTOM_VARIABLE_MATCH_NOT_RUN ) ) Assertions.assertFalse( @@ -125,8 +122,7 @@ class ControlUtilsTest : TestBase() { conditions = conditions, variables = variables, buildId = buildId, - runCondition = JobRunCondition.CUSTOM_VARIABLE_MATCH, - asCodeEnabled = true + runCondition = JobRunCondition.CUSTOM_VARIABLE_MATCH ) ) @@ -140,8 +136,7 @@ class ControlUtilsTest : TestBase() { conditions = conditions, variables = variables, buildId = buildId, - runCondition = JobRunCondition.CUSTOM_VARIABLE_MATCH_NOT_RUN, - asCodeEnabled = true + runCondition = JobRunCondition.CUSTOM_VARIABLE_MATCH_NOT_RUN ) ) Assertions.assertTrue( @@ -149,8 +144,7 @@ class ControlUtilsTest : TestBase() { conditions = conditions, variables = variables, buildId = buildId, - runCondition = JobRunCondition.CUSTOM_VARIABLE_MATCH, - asCodeEnabled = true + runCondition = JobRunCondition.CUSTOM_VARIABLE_MATCH ) ) Assertions.assertFalse( @@ -159,8 +153,7 @@ class ControlUtilsTest : TestBase() { variables = variables, buildId = buildId, runCondition = JobRunCondition.CUSTOM_CONDITION_MATCH, - customCondition = "key3=='un'", - asCodeEnabled = true + customCondition = "key3=='un'" ) ) Assertions.assertFalse( @@ -169,8 +162,7 @@ class ControlUtilsTest : TestBase() { variables = variables, buildId = buildId, runCondition = JobRunCondition.CUSTOM_CONDITION_MATCH, - customCondition = "key3==key3", - asCodeEnabled = true + customCondition = "key3==key3" ) ) Assertions.assertFalse( @@ -179,20 +171,18 @@ class ControlUtilsTest : TestBase() { variables = variables, buildId = buildId, runCondition = JobRunCondition.CUSTOM_CONDITION_MATCH, - customCondition = "true==true", - asCodeEnabled = true + customCondition = "true==true" ) ) - assertThrows { + Assertions.assertTrue( ControlUtils.checkJobSkipCondition( conditions = conditions, variables = variables, buildId = buildId, runCondition = JobRunCondition.CUSTOM_CONDITION_MATCH, - customCondition = "a==a", - asCodeEnabled = true + customCondition = "a==a" ) - } + ) } @Test @@ -408,8 +398,7 @@ class ControlUtilsTest : TestBase() { Assertions.assertFalse( ControlUtils.checkStageSkipCondition( conditions = conditions, variables = variables, buildId = buildId, - runCondition = StageRunCondition.CUSTOM_VARIABLE_MATCH_NOT_RUN, - asCodeEnabled = true + runCondition = StageRunCondition.CUSTOM_VARIABLE_MATCH_NOT_RUN ) ) @@ -418,8 +407,7 @@ class ControlUtilsTest : TestBase() { Assertions.assertTrue( ControlUtils.checkStageSkipCondition( conditions = conditions, variables = variables, buildId = buildId, - runCondition = StageRunCondition.CUSTOM_VARIABLE_MATCH_NOT_RUN, - asCodeEnabled = true + runCondition = StageRunCondition.CUSTOM_VARIABLE_MATCH_NOT_RUN ) ) @@ -428,8 +416,7 @@ class ControlUtilsTest : TestBase() { Assertions.assertFalse( ControlUtils.checkStageSkipCondition( conditions = conditions, variables = variables, buildId = buildId, - runCondition = StageRunCondition.CUSTOM_VARIABLE_MATCH, - asCodeEnabled = true + runCondition = StageRunCondition.CUSTOM_VARIABLE_MATCH ) ) @@ -438,8 +425,7 @@ class ControlUtilsTest : TestBase() { Assertions.assertTrue( ControlUtils.checkStageSkipCondition( conditions = conditions, variables = variables, buildId = buildId, - runCondition = StageRunCondition.CUSTOM_VARIABLE_MATCH, - asCodeEnabled = true + runCondition = StageRunCondition.CUSTOM_VARIABLE_MATCH ) ) @@ -448,15 +434,13 @@ class ControlUtilsTest : TestBase() { ControlUtils.checkStageSkipCondition( conditions = conditions, variables = variables, buildId = buildId, runCondition = StageRunCondition.CUSTOM_CONDITION_MATCH, - customCondition = "a==a", - asCodeEnabled = true + customCondition = "a==a" ) ) Assertions.assertFalse( ControlUtils.checkStageSkipCondition( conditions = conditions, variables = variables, buildId = buildId, - runCondition = StageRunCondition.AFTER_LAST_FINISHED, - asCodeEnabled = true + runCondition = StageRunCondition.AFTER_LAST_FINISHED ) ) } @@ -470,16 +454,14 @@ class ControlUtilsTest : TestBase() { ControlUtils.checkTaskSkip( buildId = buildId, additionalOptions = elementAdditionalOptions(runCondition = RunCondition.PRE_TASK_SUCCESS), - containerFinalStatus = fail, variables = variables, hasFailedTaskInSuccessContainer = failed, - asCodeEnabled = true + containerFinalStatus = fail, variables = variables, hasFailedTaskInSuccessContainer = failed ) ) Assertions.assertTrue( ControlUtils.checkTaskSkip( buildId = buildId, additionalOptions = elementAdditionalOptions(runCondition = RunCondition.CUSTOM_VARIABLE_MATCH), - containerFinalStatus = fail, variables = variables, hasFailedTaskInSuccessContainer = failed, - asCodeEnabled = true + containerFinalStatus = fail, variables = variables, hasFailedTaskInSuccessContainer = failed ) ) @@ -487,24 +469,21 @@ class ControlUtilsTest : TestBase() { ControlUtils.checkTaskSkip( buildId = buildId, additionalOptions = elementAdditionalOptions(runCondition = RunCondition.CUSTOM_VARIABLE_MATCH_NOT_RUN), - containerFinalStatus = fail, variables = variables, hasFailedTaskInSuccessContainer = failed, - asCodeEnabled = true + containerFinalStatus = fail, variables = variables, hasFailedTaskInSuccessContainer = failed ) ) Assertions.assertTrue( ControlUtils.checkTaskSkip( buildId = buildId, additionalOptions = elementAdditionalOptions(runCondition = RunCondition.CUSTOM_CONDITION_MATCH), - containerFinalStatus = fail, variables = variables, hasFailedTaskInSuccessContainer = failed, - asCodeEnabled = true + containerFinalStatus = fail, variables = variables, hasFailedTaskInSuccessContainer = failed ) ) Assertions.assertTrue( ControlUtils.checkTaskSkip( buildId = buildId, additionalOptions = elementAdditionalOptions(runCondition = RunCondition.OTHER_TASK_RUNNING), - containerFinalStatus = fail, variables = variables, hasFailedTaskInSuccessContainer = failed, - asCodeEnabled = true + containerFinalStatus = fail, variables = variables, hasFailedTaskInSuccessContainer = failed ) ) } @@ -519,8 +498,7 @@ class ControlUtilsTest : TestBase() { ControlUtils.checkTaskSkip( buildId = buildId, additionalOptions = elementAdditionalOptions(runCondition = RunCondition.PRE_TASK_SUCCESS), - containerFinalStatus = fail, variables = variables, hasFailedTaskInSuccessContainer = failed, - asCodeEnabled = true + containerFinalStatus = fail, variables = variables, hasFailedTaskInSuccessContainer = failed ) ) // 满足执行条件 而不跳过 @@ -531,8 +509,7 @@ class ControlUtilsTest : TestBase() { runCondition = RunCondition.CUSTOM_VARIABLE_MATCH, customVariables = mutableListOf(NameAndValue(key = "a", value = "b")) ), - containerFinalStatus = fail, variables = variables, hasFailedTaskInSuccessContainer = failed, - asCodeEnabled = true + containerFinalStatus = fail, variables = variables, hasFailedTaskInSuccessContainer = failed ) ) // 不满足执行条件 而跳过 @@ -543,8 +520,7 @@ class ControlUtilsTest : TestBase() { runCondition = RunCondition.CUSTOM_VARIABLE_MATCH, customVariables = mutableListOf(NameAndValue(key = "a", value = "a")) ), - containerFinalStatus = fail, variables = variables, hasFailedTaskInSuccessContainer = failed, - asCodeEnabled = true + containerFinalStatus = fail, variables = variables, hasFailedTaskInSuccessContainer = failed ) ) // 满足不执行的条件 而跳过 @@ -555,8 +531,7 @@ class ControlUtilsTest : TestBase() { runCondition = RunCondition.CUSTOM_VARIABLE_MATCH_NOT_RUN, customVariables = mutableListOf(NameAndValue(key = "a", value = "b")) ), - containerFinalStatus = fail, variables = variables, hasFailedTaskInSuccessContainer = failed, - asCodeEnabled = true + containerFinalStatus = fail, variables = variables, hasFailedTaskInSuccessContainer = failed ) ) // 不满足不执行的条件 而不跳过 @@ -567,8 +542,7 @@ class ControlUtilsTest : TestBase() { runCondition = RunCondition.CUSTOM_VARIABLE_MATCH_NOT_RUN, customVariables = mutableListOf(NameAndValue(key = "a", value = "a")) ), - containerFinalStatus = fail, variables = variables, hasFailedTaskInSuccessContainer = failed, - asCodeEnabled = true + containerFinalStatus = fail, variables = variables, hasFailedTaskInSuccessContainer = failed ) ) } @@ -583,8 +557,7 @@ class ControlUtilsTest : TestBase() { ControlUtils.checkTaskSkip( buildId = buildId, additionalOptions = elementAdditionalOptions(runCondition = RunCondition.PRE_TASK_SUCCESS), - containerFinalStatus = fail, variables = variables, hasFailedTaskInSuccessContainer = failed, - asCodeEnabled = true + containerFinalStatus = fail, variables = variables, hasFailedTaskInSuccessContainer = failed ) ) // 满足执行条件 而不跳过 @@ -595,8 +568,7 @@ class ControlUtilsTest : TestBase() { runCondition = RunCondition.CUSTOM_VARIABLE_MATCH, customVariables = mutableListOf(NameAndValue(key = "a", value = "b")) ), - containerFinalStatus = fail, variables = variables, hasFailedTaskInSuccessContainer = failed, - asCodeEnabled = true + containerFinalStatus = fail, variables = variables, hasFailedTaskInSuccessContainer = failed ) ) // 不满足执行条件 而跳过 @@ -607,8 +579,7 @@ class ControlUtilsTest : TestBase() { runCondition = RunCondition.CUSTOM_VARIABLE_MATCH, customVariables = mutableListOf(NameAndValue(key = "a", value = "a")) ), - containerFinalStatus = fail, variables = variables, hasFailedTaskInSuccessContainer = failed, - asCodeEnabled = true + containerFinalStatus = fail, variables = variables, hasFailedTaskInSuccessContainer = failed ) ) // 满足不执行的条件 而跳过 @@ -619,8 +590,7 @@ class ControlUtilsTest : TestBase() { runCondition = RunCondition.CUSTOM_VARIABLE_MATCH_NOT_RUN, customVariables = mutableListOf(NameAndValue(key = "a", value = "b")) ), - containerFinalStatus = fail, variables = variables, hasFailedTaskInSuccessContainer = failed, - asCodeEnabled = true + containerFinalStatus = fail, variables = variables, hasFailedTaskInSuccessContainer = failed ) ) // 不满足不执行的条件 而不跳过 @@ -631,8 +601,7 @@ class ControlUtilsTest : TestBase() { runCondition = RunCondition.CUSTOM_VARIABLE_MATCH_NOT_RUN, customVariables = mutableListOf(NameAndValue(key = "a", value = "a")) ), - containerFinalStatus = fail, variables = variables, hasFailedTaskInSuccessContainer = failed, - asCodeEnabled = true + containerFinalStatus = fail, variables = variables, hasFailedTaskInSuccessContainer = failed ) ) } @@ -648,8 +617,7 @@ class ControlUtilsTest : TestBase() { additionalOptions = null, containerFinalStatus = BuildStatus.RUNNING, variables = variables, - hasFailedTaskInSuccessContainer = true, - asCodeEnabled = true + hasFailedTaskInSuccessContainer = true ) ) @@ -662,8 +630,7 @@ class ControlUtilsTest : TestBase() { ), containerFinalStatus = BuildStatus.RUNNING, variables = variables, - hasFailedTaskInSuccessContainer = true, - asCodeEnabled = true + hasFailedTaskInSuccessContainer = true ) ) // RunCondition.PRE_TASK_FAILED_ONLY & FAIL @@ -675,8 +642,7 @@ class ControlUtilsTest : TestBase() { ), containerFinalStatus = BuildStatus.FAILED, variables = variables, - hasFailedTaskInSuccessContainer = true, - asCodeEnabled = true + hasFailedTaskInSuccessContainer = true ) ) // RunCondition.PRE_TASK_FAILED_ONLY & FAIL @@ -688,8 +654,7 @@ class ControlUtilsTest : TestBase() { ), containerFinalStatus = BuildStatus.FAILED, variables = variables, - hasFailedTaskInSuccessContainer = false, - asCodeEnabled = true + hasFailedTaskInSuccessContainer = false ) ) @@ -702,8 +667,7 @@ class ControlUtilsTest : TestBase() { ), containerFinalStatus = BuildStatus.FAILED, variables = variables, - hasFailedTaskInSuccessContainer = false, - asCodeEnabled = true + hasFailedTaskInSuccessContainer = false ) ) @@ -716,8 +680,7 @@ class ControlUtilsTest : TestBase() { ), containerFinalStatus = BuildStatus.SUCCEED, variables = variables, - hasFailedTaskInSuccessContainer = false, - asCodeEnabled = true + hasFailedTaskInSuccessContainer = false ) ) } diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/atom/TaskAtomService.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/atom/TaskAtomService.kt index 6006cc30360..7ffbce57255 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/atom/TaskAtomService.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/atom/TaskAtomService.kt @@ -45,11 +45,10 @@ import com.tencent.devops.common.pipeline.pojo.element.market.MarketBuildAtomEle import com.tencent.devops.common.pipeline.pojo.element.market.MarketBuildLessAtomElement import com.tencent.devops.common.service.utils.CommonUtils import com.tencent.devops.common.service.utils.SpringContextUtil -import com.tencent.devops.process.engine.common.VMUtils import com.tencent.devops.common.web.utils.I18nUtil import com.tencent.devops.process.constant.ProcessMessageCode.ERROR_BACKGROUND_SERVICE_RUNNING_ERROR import com.tencent.devops.process.constant.ProcessMessageCode.ERROR_BACKGROUND_SERVICE_TASK_EXECUTION -import com.tencent.devops.process.engine.control.VmOperateTaskGenerator +import com.tencent.devops.process.engine.common.VMUtils import com.tencent.devops.process.engine.exception.BuildTaskException import com.tencent.devops.process.engine.pojo.PipelineBuildTask import com.tencent.devops.process.engine.pojo.UpdateTaskInfo @@ -88,9 +87,7 @@ class TaskAtomService @Autowired(required = false) constructor( jmxElements.execute(task.taskType) var atomResponse = AtomResponse(BuildStatus.FAILED) try { - if (!VmOperateTaskGenerator.isVmAtom(task)) { - dispatchBroadCastEvent(task, ActionType.START) - } + dispatchBroadCastEvent(task, ActionType.START) // 更新状态 pipelineTaskService.updateTaskStatus( task = task, @@ -149,6 +146,7 @@ class TaskAtomService @Autowired(required = false) constructor( private fun dispatchBroadCastEvent(task: PipelineBuildTask, actionType: ActionType) { pipelineEventDispatcher.dispatch( + // 内置task启动/结束，包含 startVM、stopVM PipelineBuildStatusBroadCastEvent( source = "task-${task.taskId}", projectId = task.projectId, @@ -156,7 +154,13 @@ class TaskAtomService @Autowired(required = false) constructor( userId = task.starter, taskId = task.taskId, buildId = task.buildId, - actionType = actionType + actionType = actionType, + containerHashId = task.containerHashId, + jobId = task.jobId, + stepId = task.stepId, + atomCode = task.atomCode, + executeCount = task.executeCount, + buildStatus = task.status.name ) ) } @@ -270,9 +274,7 @@ class TaskAtomService @Autowired(required = false) constructor( logger.warn("Fail to post the task($task): ${ignored.message}") } - if (!VmOperateTaskGenerator.isVmAtom(task)) { - dispatchBroadCastEvent(task, ActionType.END) - } + dispatchBroadCastEvent(task, ActionType.END) buildLogPrinter.stopLog( buildId = task.buildId, diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/atom/task/SubPipelineCallAtom.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/atom/task/SubPipelineCallAtom.kt index 3b15e1807b1..f3a7af20d6a 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/atom/task/SubPipelineCallAtom.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/atom/task/SubPipelineCallAtom.kt @@ -200,7 +200,8 @@ class SubPipelineCallAtom constructor( } else { "syn" }, - values = startParams + values = startParams, + executeCount = task.executeCount ) if (result.isNotOk()) { diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/BuildEndControl.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/BuildEndControl.kt index d1c85ece491..4ff06c9e929 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/BuildEndControl.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/BuildEndControl.kt @@ -240,7 +240,7 @@ class BuildEndControl @Autowired constructor( buildInfo.endTime = endTime.timestampmilli() buildInfo.status = buildStatus - buildDurationTime(buildInfo.startTime!!) + buildDurationTime(buildInfo.startTime ?: 0L) callBackParentPipeline(buildInfo) // 广播结束事件 @@ -253,13 +253,16 @@ class BuildEndControl @Autowired constructor( JsonUtil.toJson(buildInfo.errorInfoList!!) } else null ), + // build 结束 PipelineBuildStatusBroadCastEvent( source = source, projectId = projectId, pipelineId = pipelineId, userId = userId, buildId = buildId, - actionType = ActionType.END + actionType = ActionType.END, + buildStatus = buildStatus.name, + executeCount = buildInfo.executeCount ), PipelineBuildWebSocketPushEvent( source = "pauseTask", diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/BuildMonitorControl.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/BuildMonitorControl.kt index 38c53f48141..2ecdfb49fe2 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/BuildMonitorControl.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/BuildMonitorControl.kt @@ -97,6 +97,15 @@ class BuildMonitorControl @Autowired constructor( fun handle(event: PipelineBuildMonitorEvent): Boolean { val buildId = event.buildId + val pipelineInfo = pipelineRepositoryService.getPipelineInfo( + projectId = event.projectId, + pipelineId = event.pipelineId, + channelCode = null + ) + if (pipelineInfo == null) { + LOG.info("ENGINE|$buildId|BUILD_MONITOR|pipeline_deleted_cancel_monitor|ec=${event.executeCount}") + return true + } val buildInfo = pipelineRuntimeService.getBuildInfo(event.projectId, buildId) if (buildInfo == null || buildInfo.isFinish()) { LOG.info("ENGINE|$buildId|BUILD_MONITOR|status=${buildInfo?.status}|ec=${event.executeCount}") @@ -121,14 +130,18 @@ class BuildMonitorControl @Autowired constructor( val minInterval = min(jobMinInt, stageMinInt) if (minInterval < min(Timeout.CONTAINER_MAX_MILLS, Timeout.STAGE_MAX_MILLS)) { - LOG.info("ENGINE|${event.buildId}|BUILD_MONITOR_CONTINUE|jobMinInt=$jobMinInt|" + - "stageMinInt=$stageMinInt|Interval=$minInterval") + LOG.info( + "ENGINE|${event.buildId}|BUILD_MONITOR_CONTINUE|jobMinInt=$jobMinInt|" + + "stageMinInt=$stageMinInt|Interval=$minInterval" + ) // 每次Check间隔不能大于10分钟，防止长时间延迟消息被大量堆积 event.delayMills = coerceAtMost10Min(minInterval).toInt() pipelineEventDispatcher.dispatch(event) } else { - LOG.info("ENGINE|${event.buildId}|BUILD_MONITOR_QUIT|jobMinInt=$jobMinInt|" + - "stageMinInt=$stageMinInt|Interval=$minInterval") + LOG.info( + "ENGINE|${event.buildId}|BUILD_MONITOR_QUIT|jobMinInt=$jobMinInt|" + + "stageMinInt=$stageMinInt|Interval=$minInterval" + ) } return true } diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/BuildStartControl.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/BuildStartControl.kt index 261134091d6..71aa1f2f0b8 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/BuildStartControl.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/BuildStartControl.kt @@ -85,6 +85,7 @@ import com.tencent.devops.process.engine.service.record.TaskBuildRecordService import com.tencent.devops.process.engine.utils.ContainerUtils import com.tencent.devops.common.pipeline.pojo.setting.PipelineRunLockType import com.tencent.devops.common.pipeline.pojo.setting.PipelineSetting +import com.tencent.devops.process.constant.ProcessMessageCode import com.tencent.devops.process.service.BuildVariableService import com.tencent.devops.process.service.scm.ScmProxyService import com.tencent.devops.process.utils.BUILD_NO @@ -266,7 +267,7 @@ class BuildStartControl @Autowired constructor( debug = buildInfo.debug ) ) - broadcastStartEvent(buildInfo) + broadcastStartEvent(buildInfo, executeCount) } else { pipelineRuntimeService.updateExecuteCount( projectId = projectId, @@ -358,10 +359,14 @@ class BuildStartControl @Autowired constructor( needShortUrl = false ) buildLogPrinter.addLine( - message = "Mode: ${setting.runLockType}," + - "concurrency for group($concurrencyGroup) " + - "and queue: ${concurrencyGroupRunning.count()}, now waiting for " + - "${concurrencyGroupRunning.first().second}", + message = I18nUtil.getCodeLanMessage( + messageCode = ProcessMessageCode.BK_BUILD_QUEUE_WAIT_FOR_CONCURRENCY, + params = arrayOf( + setting.runLockType.name, concurrencyGroup, + concurrencyGroupRunning.count().toString(), + "${concurrencyGroupRunning.first().second}" + ) + ), buildId = buildId, tag = TAG, containerHashId = JOB_ID, executeCount = executeCount, jobId = null, stepId = TAG ) @@ -395,7 +400,10 @@ class BuildStartControl @Autowired constructor( ) buildLogPrinter.addLine( - message = "Mode: ${setting.runLockType}, queue: ${buildSummaryRecord.runningCount}", + message = I18nUtil.getCodeLanMessage( + messageCode = ProcessMessageCode.BK_BUILD_QUEUE_WAIT, + params = arrayOf(setting.runLockType.name, buildSummaryRecord.runningCount.toString()) + ), buildId = buildId, tag = TAG, containerHashId = JOB_ID, executeCount = executeCount, jobId = null, stepId = TAG ) @@ -464,7 +472,7 @@ class BuildStartControl @Autowired constructor( } } - private fun PipelineBuildStartEvent.broadcastStartEvent(buildInfo: BuildInfo) { + private fun PipelineBuildStartEvent.broadcastStartEvent(buildInfo: BuildInfo, executeCount: Int) { pipelineEventDispatcher.dispatch( // 广播构建即将启动消息给订阅者 PipelineBuildStartBroadCastEvent( @@ -476,14 +484,16 @@ class BuildStartControl @Autowired constructor( startTime = buildInfo.startTime, triggerType = buildInfo.trigger ), - // 根据状态做响应的扩展广播消息给订阅者 + // build 启动，根据状态做响应的扩展广播消息给订阅者 PipelineBuildStatusBroadCastEvent( source = source, projectId = projectId, pipelineId = pipelineId, userId = userId, buildId = buildId, - actionType = ActionType.START + actionType = ActionType.START, + executeCount = executeCount, + buildStatus = BuildStatus.RUNNING.name ) ) } diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/MutexControl.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/MutexControl.kt index a7248f3ca27..12f7cfc1f1d 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/MutexControl.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/MutexControl.kt @@ -54,6 +54,7 @@ import com.tencent.devops.process.constant.ProcessMessageCode.BK_RELEASE_LOCK import com.tencent.devops.process.engine.common.Timeout import com.tencent.devops.process.engine.common.VMUtils import com.tencent.devops.process.engine.pojo.PipelineBuildContainer +import com.tencent.devops.process.engine.service.EngineConfigService import com.tencent.devops.process.engine.service.PipelineContainerService import com.tencent.devops.process.engine.service.record.ContainerBuildRecordService import com.tencent.devops.process.utils.PipelineVarUtil @@ -70,13 +71,13 @@ class MutexControl @Autowired constructor( private val redisOperation: RedisOperation, private val pipelineUrlBean: PipelineUrlBean, private val containerBuildRecordService: ContainerBuildRecordService, - private val pipelineContainerService: PipelineContainerService + private val pipelineContainerService: PipelineContainerService, + private val engineConfigService: EngineConfigService ) { companion object { private const val DELIMITERS = "_" const val SECOND_TO_PRINT = 19 - const val MUTEX_MAX_QUEUE = 10 private val LOG = LoggerFactory.getLogger(MutexControl::class.java) private fun getMutexContainerId(buildId: String, containerId: String) = "${buildId}$DELIMITERS$containerId" private fun getBuildIdAndContainerId(mutexId: String): List = mutexId.split(DELIMITERS) @@ -106,7 +107,7 @@ class MutexControl @Autowired constructor( // 超时时间限制，0表示排队不等待直接超时 val timeOut = parseTimeoutVar(mutexGroup.timeout, mutexGroup.timeoutVar, variables) // 排队任务数量限制，0表示不排队 - val queue = mutexGroup.queue.coerceAtLeast(0).coerceAtMost(MUTEX_MAX_QUEUE) + val queue = mutexGroup.queue.coerceAtLeast(0).coerceAtMost(engineConfigService.getMutexMaxQueue()) // 替换环境变量 val mutexLockedGroup = if (!mutexGroup.mutexGroupName.isNullOrBlank()) { EnvUtils.parseEnv(mutexGroup.mutexGroupName, variables) diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/container/impl/AgentReuseMutexCmd.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/container/impl/AgentReuseMutexCmd.kt index 45d1f6e468e..5722b45fccd 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/container/impl/AgentReuseMutexCmd.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/container/impl/AgentReuseMutexCmd.kt @@ -23,6 +23,7 @@ import com.tencent.devops.process.engine.control.command.CmdFlowState import com.tencent.devops.process.engine.control.command.container.ContainerCmd import com.tencent.devops.process.engine.control.command.container.ContainerContext import com.tencent.devops.process.engine.pojo.PipelineBuildContainer +import com.tencent.devops.process.engine.service.EngineConfigService import com.tencent.devops.process.engine.service.record.ContainerBuildRecordService import com.tencent.devops.process.service.BuildVariableService import org.slf4j.LoggerFactory @@ -38,12 +39,13 @@ class AgentReuseMutexCmd @Autowired constructor( private val buildLogPrinter: BuildLogPrinter, private val containerBuildRecordService: ContainerBuildRecordService, private val pipelineUrlBean: PipelineUrlBean, - private val buildVariableService: BuildVariableService + private val buildVariableService: BuildVariableService, + private val engineConfigService: EngineConfigService ) : ContainerCmd { override fun canExecute(commandContext: ContainerContext): Boolean { return commandContext.cmdFlowState == CmdFlowState.CONTINUE && - !commandContext.buildStatus.isFinish() && - commandContext.container.controlOption.agentReuseMutex != null + !commandContext.buildStatus.isFinish() && + commandContext.container.controlOption.agentReuseMutex != null } override fun execute(commandContext: ContainerContext) { @@ -57,22 +59,15 @@ class AgentReuseMutexCmd @Autowired constructor( * 互斥情况存在三种 * 1、依赖某个AgentId，直接往下执行即可 * 2、Root节点，即没有复用节点的节点，根据类型先拿取锁 - * 3、Reuse节点，但没有复用节点，可能是因为和复用节点同级，或存在和复用节点先后顺序不明确的， - * 这种先拿取复用的JobId看有没有，没有就按root节点的逻辑走 + * 3、Reuse节点，但没有复用节点，可能是因为和复用节点同级，或存在和复用节点先后顺序不明确的，就按root节点的逻辑走 */ private fun doExecute(commandContext: ContainerContext) { var mutex = commandContext.container.controlOption.agentReuseMutex!! + // 复用的不用参与锁逻辑 if (!mutex.type.needEngineLock()) { commandContext.cmdFlowState = CmdFlowState.CONTINUE return } - // 如果有依赖Job且不是依赖类型的可以先拿一下上下文看看有没有已经写入了，如果写入了可以直接跳过 - if (mutex.reUseJobId != null && - commandContext.variables.containsKey(AgentReuseMutex.genAgentContextKey(mutex.reUseJobId!!)) - ) { - commandContext.cmdFlowState = CmdFlowState.CONTINUE - return - } // 极端情况上下文没有写入，且agentId还是空，理论上不会有，逻辑上出现了就失败 if (mutex.agentOrEnvId.isNullOrBlank()) { return agentIdNullError(commandContext, mutex, null) @@ -83,7 +78,7 @@ class AgentReuseMutexCmd @Autowired constructor( AgentReuseMutexType.AGENT_ID, AgentReuseMutexType.AGENT_NAME -> { acquireMutex(commandContext, mutex) } - + // 环境的因为无法确定节点不参与锁 else -> { commandContext.cmdFlowState = CmdFlowState.CONTINUE } @@ -102,7 +97,7 @@ class AgentReuseMutexCmd @Autowired constructor( // 超时时间限制，0表示排队不等待直接超时 val timeOut = MutexControl.parseTimeoutVar(mutex.timeout, mutex.timeoutVar, variables) // 排队任务数量限制，0表示不排队 - val queue = mutex.queue.coerceAtLeast(0).coerceAtMost(MutexControl.MUTEX_MAX_QUEUE) + val queue = mutex.queue.coerceAtLeast(0).coerceAtMost(engineConfigService.getMutexMaxQueue()) // 替换环境变量 var runtimeAgentOrEnvId = if (!mutex.agentOrEnvId.isNullOrBlank()) { EnvUtils.parseEnv(mutex.agentOrEnvId, variables) @@ -158,21 +153,23 @@ class AgentReuseMutexCmd @Autowired constructor( commandContext.cmdFlowState = CmdFlowState.LOOP // 循环消息命令 延时10秒钟 } - ContainerMutexStatus.FIRST_LOG -> { // 增加可视化的互斥状态打印 + ContainerMutexStatus.FIRST_LOG -> { // 增加可视化的互斥状态打印，注：这里进行了Job状态流转！ commandContext.latestSummary = "agent_reuse_mutex_print" commandContext.cmdFlowState = CmdFlowState.LOOP } - else -> { // 正常运行 - commandContext.cmdFlowState = CmdFlowState.CONTINUE // 检查通过，继续向下执行 + // 正常运行 + else -> { + // 检查通过，继续向下执行 + commandContext.cmdFlowState = CmdFlowState.CONTINUE } } } else if (commandContext.container.status.isFinish()) { // 对于存在重放的结束消息做闭环 val event = commandContext.event LOG.info( - "ENGINE|${event.buildId}|${event.source}|status=${commandContext.container.status}" + - "|concurrent_container_event" + "AGENT_REUSE|ENGINE|${event.buildId}|${event.source}|status=${commandContext.container.status}" + + "|concurrent_container_event" ) releaseContainerMutex( @@ -189,21 +186,21 @@ class AgentReuseMutexCmd @Autowired constructor( } private fun doAcquireMutex(container: PipelineBuildContainer, mutex: AgentReuseMutex): ContainerMutexStatus { + // 对于AgentId这种需要写入上下文，防止最后没有被执行导致兜底无法解锁，排队中也可能被取消导致无法退出队列 + // 只要不是依赖节点，根节点和同级依赖节点都要写入，防止出现被同级env节点获取到节点锁但没写入依赖id节点就无法获取到AgentId + buildVariableService.setVariable( + projectId = container.projectId, + pipelineId = container.pipelineId, + buildId = container.buildId, + varName = AgentReuseMutex.genAgentContextKey(mutex.reUseJobId ?: mutex.jobId), + varValue = mutex.runtimeAgentOrEnvId!!, + readOnly = true, + rewriteReadOnly = true + ) + val res = tryAgentLockOrQueue(container, mutex) return if (res) { LOG.info("ENGINE|${container.buildId}|AGENT_REUSE_LOCK_SUCCESS|${mutex.type}") - - // 对于AgentId这种获取到锁就需要写入上下文，防止最后没有被执行导致兜底无法解锁 - // 只要不是依赖节点，根节点和同级依赖节点都要写入，防止出现被同级env节点获取到节点锁但没写入依赖id节点就无法获取到AgentId - buildVariableService.setVariable( - projectId = container.projectId, - pipelineId = container.pipelineId, - buildId = container.buildId, - varName = AgentReuseMutex.genAgentContextKey(mutex.reUseJobId ?: mutex.jobId), - varValue = mutex.runtimeAgentOrEnvId!!, - readOnly = true - ) - // 抢到锁则可以继续运行，并退出队列 quitMutexQueue( projectId = container.projectId, @@ -235,13 +232,13 @@ class AgentReuseMutexCmd @Autowired constructor( private fun tryAgentLockOrQueue(container: PipelineBuildContainer, mutex: AgentReuseMutex): Boolean { val lockKey = AgentReuseMutex.genAgentReuseMutexLockKey(container.projectId, mutex.runtimeAgentOrEnvId!!) val lockValue = container.buildId - // 过期时间按整个流水线过期时间为准 + // 过期时间按整个流水线过期时间为准，7天 val expireSec = AgentReuseMutex.AGENT_LOCK_TIMEOUT val containerMutexLock = RedisLockByValue( redisOperation = redisOperation, lockKey = lockKey, lockValue = lockValue, - expiredTimeInSeconds = AgentReuseMutex.AGENT_LOCK_TIMEOUT + expiredTimeInSeconds = expireSec ) // 获取到锁的内容 val lv = redisOperation.get(lockKey) @@ -275,9 +272,9 @@ class AgentReuseMutexCmd @Autowired constructor( if (lockResult) { mutex.linkTip?.let { redisOperation.set( - AgentReuseMutex.genAgentReuseMutexLinkTipKey(container.buildId), - mutex.linkTip!!, - expireSec + key = AgentReuseMutex.genAgentReuseMutexLinkTipKey(container.buildId), + value = mutex.linkTip!!, + expiredInSecond = expireSec ) } logAgentMutex( @@ -308,7 +305,7 @@ class AgentReuseMutexCmd @Autowired constructor( containerVar = emptyMap(), buildStatus = null, timestamps = mapOf( BuildTimestampType.JOB_AGENT_REUSE_MUTEX_QUEUE to - BuildRecordTimeStamp(null, LocalDateTime.now().timestampmilli()) + BuildRecordTimeStamp(null, LocalDateTime.now().timestampmilli()) ) ) } @@ -345,9 +342,9 @@ class AgentReuseMutexCmd @Autowired constructor( // 排队等待时间为0的时候，立即超时, 退出队列，并失败, 没有就继续在队列中,timeOut时间为分钟 if (mutex.timeout == 0 || timeDiff > TimeUnit.MINUTES.toSeconds(mutex.timeout.toLong())) { val desc = "${ - if (mutex.timeoutVar.isNullOrBlank()) { - "[${mutex.timeout} minutes]" - } else " timeoutVar[${mutex.timeoutVar}] setup to [${mutex.timeout} minutes]" + if (mutex.timeoutVar.isNullOrBlank()) { + "[${mutex.timeout} minutes]" + } else " timeoutVar[${mutex.timeoutVar}] setup to [${mutex.timeout} minutes]" } " logAgentMutex( container, mutex, lockedBuildId, @@ -441,7 +438,7 @@ class AgentReuseMutexCmd @Autowired constructor( containerVar = emptyMap(), buildStatus = null, timestamps = mapOf( BuildTimestampType.JOB_MUTEX_QUEUE to - BuildRecordTimeStamp(LocalDateTime.now().timestampmilli(), null) + BuildRecordTimeStamp(LocalDateTime.now().timestampmilli(), null) ) ) } @@ -455,7 +452,7 @@ class AgentReuseMutexCmd @Autowired constructor( executeCount: Int? ) { LOG.info( - "[$buildId]|RELEASE_MUTEX_LOCK|project=$projectId|$runtimeAgentOrEnvId" + "[$buildId]|AGENT_REUSE_RELEASE_MUTEX_LOCK|project=$projectId|$runtimeAgentOrEnvId" ) // 删除tip redisOperation.delete(AgentReuseMutex.genAgentReuseMutexLinkTipKey(buildId)) @@ -512,10 +509,10 @@ class AgentReuseMutexCmd @Autowired constructor( messageCode = ProcessMessageCode.BK_LOCKED, language = I18nUtil.getDefaultLocaleLanguage() ) + ": $linkTip" + - I18nUtil.getCodeLanMessage( - messageCode = ProcessMessageCode.BK_CLICK, - language = I18nUtil.getDefaultLocaleLanguage() - ) + " | $msg" + I18nUtil.getCodeLanMessage( + messageCode = ProcessMessageCode.BK_CLICK, + language = I18nUtil.getDefaultLocaleLanguage() + ) + " | $msg" } else { I18nUtil.getCodeLanMessage( messageCode = ProcessMessageCode.BK_CURRENT, diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/container/impl/CheckConditionalSkipContainerCmd.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/container/impl/CheckConditionalSkipContainerCmd.kt index aa5821f1f8d..54cc09b8d66 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/container/impl/CheckConditionalSkipContainerCmd.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/container/impl/CheckConditionalSkipContainerCmd.kt @@ -128,14 +128,13 @@ class CheckConditionalSkipContainerCmd constructor( buildId = container.buildId, runCondition = jobControlOption.runCondition, customCondition = jobControlOption.customCondition, - message = message, - asCodeEnabled = containerContext.pipelineAsCodeEnabled == true + message = message ) } if (message.isNotBlank()) { // #6366 增加日志明确展示跳过的原因 - buildLogPrinter.addDebugLine( + buildLogPrinter.addWarnLine( executeCount = containerContext.executeCount, tag = VMUtils.genStartVMTaskId(container.containerId), buildId = container.buildId, diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/container/impl/CheckMutexContainerCmd.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/container/impl/CheckMutexContainerCmd.kt index 944cf71a2d9..6099a47fc63 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/container/impl/CheckMutexContainerCmd.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/container/impl/CheckMutexContainerCmd.kt @@ -94,7 +94,7 @@ class CheckMutexContainerCmd( commandContext.cmdFlowState = CmdFlowState.LOOP // 循环消息命令 延时10秒钟 } - ContainerMutexStatus.FIRST_LOG -> { // #5454 增加可视化的互斥状态打印 + ContainerMutexStatus.FIRST_LOG -> { // #5454 增加可视化的互斥状态打印，注：这里进行了Job状态流转！ commandContext.latestSummary = "mutex_print" commandContext.cmdFlowState = CmdFlowState.LOOP } diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/container/impl/ContainerCmdLoop.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/container/impl/ContainerCmdLoop.kt index 4dbc5e2132e..ecdc00716cc 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/container/impl/ContainerCmdLoop.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/container/impl/ContainerCmdLoop.kt @@ -57,7 +57,9 @@ class ContainerCmdLoop( commandContext.event.copy(delayMills = DEFAULT_LOOP_TIME_MILLS, source = commandContext.latestSummary) ) // #5454 增加可视化的互斥状态打印 - if (commandContext.latestSummary == "mutex_print") { + if (commandContext.latestSummary == "mutex_print" || + commandContext.latestSummary == "agent_reuse_mutex_print" + ) { commandContext.cmdFlowState = CmdFlowState.FINALLY } } diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/container/impl/StartActionTaskContainerCmd.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/container/impl/StartActionTaskContainerCmd.kt index 43cff5e0a74..36e512a7b52 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/container/impl/StartActionTaskContainerCmd.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/container/impl/StartActionTaskContainerCmd.kt @@ -30,6 +30,7 @@ package com.tencent.devops.process.engine.control.command.container.impl import com.tencent.devops.common.api.pojo.ErrorType import com.tencent.devops.common.event.dispatcher.pipeline.PipelineEventDispatcher import com.tencent.devops.common.event.enums.ActionType +import com.tencent.devops.common.event.pojo.pipeline.PipelineBuildStatusBroadCastEvent import com.tencent.devops.common.expression.ExpressionParseException import com.tencent.devops.common.log.utils.BuildLogPrinter import com.tencent.devops.common.pipeline.enums.BuildStatus @@ -94,6 +95,9 @@ class StartActionTaskContainerCmd( commandContext.buildStatus = BuildStatus.SUCCEED } val waitToDoTask = findTask(commandContext) + if (waitToDoTask != null && TaskUtils.isStartVMTask(waitToDoTask)) { + sendJobStartCallback(commandContext) + } if (waitToDoTask == null) { // 非fast kill的强制终止时到最后无任务，最终状态必定是FAILED val fastKill = FastKillUtils.isFastKillCode(commandContext.event.errorCode) if (!fastKill && actionType.isTerminate() && !commandContext.buildStatus.isFailure()) { @@ -115,6 +119,28 @@ class StartActionTaskContainerCmd( } } + private fun sendJobStartCallback(commandContext: ContainerContext) { + with(commandContext.container) { + pipelineEventDispatcher.dispatch( + // job 启动 + PipelineBuildStatusBroadCastEvent( + source = "StartActionTaskContainerCmd", + projectId = projectId, + pipelineId = pipelineId, + userId = commandContext.event.userId, + buildId = buildId, + actionType = ActionType.START, + stageId = stageId, + containerHashId = containerHashId, + jobId = jobId, + stepId = null, + executeCount = executeCount, + buildStatus = commandContext.buildStatus.name + ) + ) + } + } + private fun setContextBuildStatus(commandContext: ContainerContext) { val container = commandContext.container val runEvenCancelTaskIdKey = ContainerUtils.getContainerRunEvenCancelTaskKey( @@ -424,7 +450,7 @@ class StartActionTaskContainerCmd( if (message.isNotBlank()) { // #6366 增加日志明确展示跳过的原因 // 打印构建日志--DEBUG级别日志，平时隐藏 - buildLogPrinter.addDebugLine( + buildLogPrinter.addWarnLine( executeCount = containerContext.executeCount, tag = taskId, buildId = buildId, containerHashId = containerHashId, message = message.toString(), jobId = null, stepId = stepId @@ -464,8 +490,7 @@ class StartActionTaskContainerCmd( containerFinalStatus = containerContext.buildStatus, variables = contextMap, hasFailedTaskInSuccessContainer = hasFailedTaskInSuccessContainer, - message = message, - asCodeEnabled = containerContext.pipelineAsCodeEnabled == true + message = message ) } @@ -480,8 +505,7 @@ class StartActionTaskContainerCmd( containerFinalStatus = containerContext.buildStatus, variables = contextMap, hasFailedTaskInSuccessContainer = hasFailedTaskInSuccessContainer, - message = message, - asCodeEnabled = containerContext.pipelineAsCodeEnabled == true + message = message ) if (LOG.isDebugEnabled) { LOG.debug("ENGINE|$buildId|CHECK_QUICK_SKIP|$stageId|j($containerId)|${it.taskName}|$skip") diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/container/impl/UpdateStateContainerCmdFinally.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/container/impl/UpdateStateContainerCmdFinally.kt index f3da23c0b86..440ddd189ea 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/container/impl/UpdateStateContainerCmdFinally.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/container/impl/UpdateStateContainerCmdFinally.kt @@ -29,6 +29,7 @@ package com.tencent.devops.process.engine.control.command.container.impl import com.tencent.devops.common.event.dispatcher.pipeline.PipelineEventDispatcher import com.tencent.devops.common.event.enums.ActionType +import com.tencent.devops.common.event.pojo.pipeline.PipelineBuildStatusBroadCastEvent import com.tencent.devops.common.log.utils.BuildLogPrinter import com.tencent.devops.common.pipeline.enums.BuildStatus import com.tencent.devops.common.redis.RedisOperation @@ -91,15 +92,32 @@ class UpdateStateContainerCmdFinally( if (matrixGroupId.isNullOrBlank()) { sendBackStage(commandContext = commandContext) } else { - pipelineEventDispatcher.dispatch( - commandContext.event.copy( - actionType = ActionType.REFRESH, - containerId = matrixGroupId, - containerHashId = null, - source = commandContext.latestSummary, - reason = "Matrix(${commandContext.container.containerId}) inner container finished" + with(commandContext.event) { + pipelineEventDispatcher.dispatch( + commandContext.event.copy( + actionType = ActionType.REFRESH, + containerId = matrixGroupId, + containerHashId = null, + source = commandContext.latestSummary, + reason = "Matrix(${commandContext.container.containerId}) inner container finished" + ), + // matrix job 结束 + PipelineBuildStatusBroadCastEvent( + source = "StartActionTaskContainerCmd", + projectId = projectId, + pipelineId = pipelineId, + userId = userId, + buildId = buildId, + actionType = ActionType.END, + stageId = stageId, + containerHashId = containerHashId, + jobId = commandContext.container.jobId, + stepId = null, + executeCount = executeCount, + buildStatus = commandContext.buildStatus.name + ) ) - ) + } } } } @@ -211,6 +229,21 @@ class UpdateStateContainerCmdFinally( buildId = buildId, stageId = stageId, actionType = ActionType.REFRESH + ), + // job 结束 + PipelineBuildStatusBroadCastEvent( + source = "StartActionTaskContainerCmd", + projectId = projectId, + pipelineId = pipelineId, + userId = userId, + buildId = buildId, + actionType = ActionType.END, + stageId = stageId, + containerHashId = containerHashId, + jobId = commandContext.container.jobId, + stepId = null, + executeCount = executeCount, + buildStatus = commandContext.buildStatus.name ) ) diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/stage/impl/CheckConditionalSkipStageCmd.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/stage/impl/CheckConditionalSkipStageCmd.kt index 3b07f10138c..21df1004b61 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/stage/impl/CheckConditionalSkipStageCmd.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/stage/impl/CheckConditionalSkipStageCmd.kt @@ -99,6 +99,7 @@ class CheckConditionalSkipStageCmd constructor( // condition check val variables = commandContext.variables var skip = false + val message = StringBuilder() if (controlOption != null) { val conditions = controlOption.customVariables ?: emptyList() val contextMap = pipelineContextService.buildContext( @@ -116,9 +117,19 @@ class CheckConditionalSkipStageCmd constructor( variables = variables.plus(contextMap), buildId = stage.buildId, runCondition = controlOption.runCondition, - customCondition = controlOption.customCondition, - asCodeEnabled = commandContext.pipelineAsCodeEnabled == true + customCondition = controlOption.customCondition ) // #6366 增加日志明确展示跳过的原因 stage 没有相关可展示的地方，暂时不加 + if (message.isNotBlank()) { + // #6366 增加日志明确展示跳过的原因 + buildLogPrinter.addWarnLine( + executeCount = commandContext.executeCount, + tag = "", + buildId = stage.buildId, + message = message.toString(), + jobId = null, + stepId = null + ) + } } if (skip) { LOG.info("ENGINE|${event.buildId}|${event.source}|STAGE_CONDITION_SKIP|${event.stageId}|$controlOption") diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/stage/impl/CheckPauseReviewStageCmd.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/stage/impl/CheckPauseReviewStageCmd.kt index 16b77a5af9e..dc4fcb29cb9 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/stage/impl/CheckPauseReviewStageCmd.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/stage/impl/CheckPauseReviewStageCmd.kt @@ -64,7 +64,9 @@ class CheckPauseReviewStageCmd( val event = commandContext.event // 处于等待中，遇到停止/取消等行为直接结束，因为本Stage还未进入 - if (event.actionType.isEnd() && commandContext.buildStatus.isPause()) { + if (event.actionType.isEnd() && + (commandContext.buildStatus.isPause() || commandContext.buildStatus.isReadyToRun()) + ) { commandContext.buildStatus = BuildStatus.CANCELED commandContext.cmdFlowState = CmdFlowState.FINALLY LOG.info("ENGINE|${event.buildId}|${event.source}|STAGE_CANCEL|${event.stageId}") diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/stage/impl/StartContainerStageCmd.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/stage/impl/StartContainerStageCmd.kt index 2dbf81538a1..fbdef7c056f 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/stage/impl/StartContainerStageCmd.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/stage/impl/StartContainerStageCmd.kt @@ -110,6 +110,7 @@ class StartContainerStageCmd( private fun sendStageStartCallback(commandContext: StageContext) { pipelineEventDispatcher.dispatch( + // stage 启动 PipelineBuildStatusBroadCastEvent( source = "StartContainerStageCmd", projectId = commandContext.stage.projectId, @@ -117,7 +118,9 @@ class StartContainerStageCmd( userId = commandContext.event.userId, buildId = commandContext.stage.buildId, actionType = ActionType.START, - stageId = commandContext.stage.stageId + stageId = commandContext.stage.stageId, + executeCount = commandContext.executeCount, + buildStatus = commandContext.buildStatus.name ) ) } @@ -240,10 +243,10 @@ class StartContainerStageCmd( return } val lock = RedisLockByValue( - redisOperation, - AgentReuseMutex.genAgentReuseMutexLockKey(stage.projectId, agent), - stage.buildId, - AgentReuseMutex.AGENT_LOCK_TIMEOUT + redisOperation = redisOperation, + lockKey = AgentReuseMutex.genAgentReuseMutexLockKey(stage.projectId, agent), + lockValue = stage.buildId, + expiredTimeInSeconds = AgentReuseMutex.AGENT_LOCK_TIMEOUT ) lock.unlock() } diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/stage/impl/UpdateStateForStageCmdFinally.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/stage/impl/UpdateStateForStageCmdFinally.kt index a8882d563a7..8e9959ced81 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/stage/impl/UpdateStateForStageCmdFinally.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/control/command/stage/impl/UpdateStateForStageCmdFinally.kt @@ -47,11 +47,10 @@ import com.tencent.devops.process.engine.pojo.event.PipelineBuildStageEvent import com.tencent.devops.process.engine.service.PipelineContainerService import com.tencent.devops.process.engine.service.PipelineRuntimeService import com.tencent.devops.process.engine.service.PipelineStageService -import com.tencent.devops.process.engine.service.detail.StageBuildDetailService import com.tencent.devops.process.engine.service.record.StageBuildRecordService +import java.time.LocalDateTime import org.slf4j.LoggerFactory import org.springframework.stereotype.Service -import java.time.LocalDateTime /** * 每一个Stage结束后续命令处理 @@ -62,7 +61,6 @@ class UpdateStateForStageCmdFinally( private val pipelineStageService: PipelineStageService, private val pipelineRuntimeService: PipelineRuntimeService, private val pipelineContainerService: PipelineContainerService, - private val stageBuildDetailService: StageBuildDetailService, private val stageBuildRecordService: StageBuildRecordService, private val pipelineEventDispatcher: PipelineEventDispatcher, private val buildLogPrinter: BuildLogPrinter, @@ -97,7 +95,7 @@ class UpdateStateForStageCmdFinally( pipelineStageService.pauseStage(stage, commandContext.debug) } else { nextOrFinish(event, stage, commandContext, false) - sendStageEndCallBack(stage, event) + sendStageEndCallBack(commandContext) } } else if (commandContext.buildStatus.isFinish()) { // 当前Stage结束 if (commandContext.buildStatus == BuildStatus.SKIP) { // 跳过 @@ -106,15 +104,19 @@ class UpdateStateForStageCmdFinally( pipelineStageService.refreshCheckStageStatus(userId = event.userId, buildStage = stage, inOrOut = false) } nextOrFinish(event, stage, commandContext, commandContext.buildStatus.isSuccess()) - sendStageEndCallBack(stage, event) + sendStageEndCallBack(commandContext) } } - private fun sendStageEndCallBack(stage: PipelineBuildStage, event: PipelineBuildStageEvent) { + private fun sendStageEndCallBack(commandContext: StageContext) { + val event = commandContext.event + val stage = commandContext.stage pipelineEventDispatcher.dispatch( + // stage 结束 PipelineBuildStatusBroadCastEvent( source = "UpdateStateForStageCmdFinally", projectId = stage.projectId, pipelineId = stage.pipelineId, - userId = event.userId, buildId = stage.buildId, stageId = stage.stageId, actionType = ActionType.END + userId = event.userId, buildId = stage.buildId, stageId = stage.stageId, actionType = ActionType.END, + buildStatus = commandContext.buildStatus.name, executeCount = stage.executeCount ) ) } @@ -151,7 +153,23 @@ class UpdateStateForStageCmdFinally( return finishBuild(commandContext = commandContext) } - event.actionType = ActionType.START // final 需要执行 + if (nextStage.controlOption?.finally == true) { + val pendingStages = pipelineStageService.getPendingStages(event.projectId, event.buildId) + .filter { it.stageId != nextStage.stageId } + pendingStages.forEach { pendingStage -> + pendingStage.status = BuildStatus.UNEXEC + stageBuildRecordService.updateStageStatus( + projectId = pendingStage.projectId, + pipelineId = pendingStage.pipelineId, + buildId = pendingStage.buildId, + stageId = pendingStage.stageId, + executeCount = pendingStage.executeCount, + buildStatus = BuildStatus.UNEXEC + ) + } + pipelineStageService.batchUpdate(transactionContext = null, stageList = pendingStages) + event.actionType = ActionType.START // final 需要执行 + } } else { nextStage = pipelineStageService.getNextStage( projectId = event.projectId, @@ -197,9 +215,11 @@ class UpdateStateForStageCmdFinally( event.source == BS_QUALITY_PASS_STAGE -> { qualityCheckOutPass(commandContext) } + event.source == BS_QUALITY_ABORT_STAGE || event.actionType.isEnd() -> { qualityCheckOutFailed(commandContext) } + else -> { val checkStatus = pipelineStageService.checkStageQuality( event = event, @@ -211,11 +231,13 @@ class UpdateStateForStageCmdFinally( BuildStatus.QUALITY_CHECK_PASS -> { qualityCheckOutPass(commandContext) } + BuildStatus.QUALITY_CHECK_WAIT -> { // #5246 如果设置了把关人则卡在运行状态等待审核 qualityCheckOutNeedReview(commandContext) needBreak = true } + else -> { qualityCheckOutFailed(commandContext) } diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/init/BuildEngineExtendConfiguration.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/init/BuildEngineExtendConfiguration.kt index 456afb58272..201b7971c1d 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/init/BuildEngineExtendConfiguration.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/init/BuildEngineExtendConfiguration.kt @@ -30,6 +30,7 @@ package com.tencent.devops.process.engine.init import com.tencent.devops.common.event.dispatcher.pipeline.mq.MQ import com.tencent.devops.common.event.dispatcher.pipeline.mq.Tools import com.tencent.devops.process.engine.listener.run.finish.SubPipelineBuildFinishListener +import com.tencent.devops.process.engine.listener.run.start.SubPipelineBuildQueueListener import org.springframework.amqp.core.Binding import org.springframework.amqp.core.BindingBuilder import org.springframework.amqp.core.DirectExchange @@ -160,4 +161,38 @@ class BuildEngineExtendConfiguration { maxConcurrency = 10 ) } + + @Bean + fun subPipelineBuildQueue(): Queue { + return Queue(MQ.QUEUE_PIPELINE_BUILD_QUEUE_SUBPIPEINE) + } + + @Bean + fun subPipelineBuildQueueBind( + @Autowired subPipelineBuildQueue: Queue, + @Autowired pipelineBuildQueueFanoutExchange: FanoutExchange + ): Binding { + return BindingBuilder.bind(subPipelineBuildQueue).to(pipelineBuildQueueFanoutExchange) + } + + @Bean + fun subPipelineBuildStartListenerContainer( + @Autowired connectionFactory: ConnectionFactory, + @Autowired subPipelineBuildQueue: Queue, + @Autowired rabbitAdmin: RabbitAdmin, + @Autowired buildListener: SubPipelineBuildQueueListener, + @Autowired messageConverter: Jackson2JsonMessageConverter + ): SimpleMessageListenerContainer { + return Tools.createSimpleMessageListenerContainer( + connectionFactory = connectionFactory, + queue = subPipelineBuildQueue, + rabbitAdmin = rabbitAdmin, + buildListener = buildListener, + messageConverter = messageConverter, + startConsumerMinInterval = 10000, + consecutiveActiveTrigger = 5, + concurrency = 1, + maxConcurrency = 10 + ) + } } diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/listener/run/PipelineTaskPauseListener.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/listener/run/PipelineTaskPauseListener.kt index 7b8d9323749..b8f29ad4c17 100644 --- a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/listener/run/PipelineTaskPauseListener.kt +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/listener/run/PipelineTaskPauseListener.kt @@ -217,15 +217,22 @@ class PipelineTaskPauseListener @Autowired constructor( executeCount = task.executeCount, containerType = containerRecord?.containerType ?: "vmBuild" ), + // pause task 结束 PipelineBuildStatusBroadCastEvent( source = "pauseCancel-${task.containerId}-${task.buildId}", projectId = task.projectId, pipelineId = task.pipelineId, userId = task.starter, buildId = task.buildId, - taskId = null, - stageId = null, - actionType = ActionType.END + taskId = task.taskId, + stageId = task.stageId, + actionType = ActionType.END, + containerHashId = task.containerHashId, + jobId = task.jobId, + stepId = task.stepId, + atomCode = task.atomCode, + executeCount = task.executeCount, + buildStatus = BuildStatus.CANCELED.name ) ) } diff --git a/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/listener/run/start/SubPipelineBuildQueueListener.kt b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/listener/run/start/SubPipelineBuildQueueListener.kt new file mode 100644 index 00000000000..6d92f7e4a2b --- /dev/null +++ b/src/backend/ci/core/process/biz-engine/src/main/kotlin/com/tencent/devops/process/engine/listener/run/start/SubPipelineBuildQueueListener.kt @@ -0,0 +1,46 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.process.engine.listener.run.start + +import com.tencent.devops.common.event.dispatcher.pipeline.PipelineEventDispatcher +import com.tencent.devops.common.event.listener.pipeline.BaseListener +import com.tencent.devops.common.event.pojo.pipeline.PipelineBuildQueueBroadCastEvent +import com.tencent.devops.process.service.SubPipelineStatusService +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Component + +@Component +class SubPipelineBuildQueueListener @Autowired constructor( + private val subPipelineStatusService: SubPipelineStatusService, + pipelineEventDispatcher: PipelineEventDispatcher +) : BaseListener(pipelineEventDispatcher) { + + override fun run(event: PipelineBuildQueueBroadCastEvent) { + subPipelineStatusService.onBuildQueue(event) + } +} diff --git a/src/backend/ci/core/process/biz-engine/src/test/kotlin/com/tencent/devops/process/engine/control/MutexControlTest.kt b/src/backend/ci/core/process/biz-engine/src/test/kotlin/com/tencent/devops/process/engine/control/MutexControlTest.kt index 6c36b7c22f1..0c9997e5670 100644 --- a/src/backend/ci/core/process/biz-engine/src/test/kotlin/com/tencent/devops/process/engine/control/MutexControlTest.kt +++ b/src/backend/ci/core/process/biz-engine/src/test/kotlin/com/tencent/devops/process/engine/control/MutexControlTest.kt @@ -35,6 +35,8 @@ import com.tencent.devops.common.pipeline.option.JobControlOption import com.tencent.devops.common.redis.RedisOperation import com.tencent.devops.process.engine.pojo.PipelineBuildContainer import com.tencent.devops.process.engine.pojo.PipelineBuildContainerControlOption +import com.tencent.devops.process.engine.service.EngineConfigService +import io.mockk.every import io.mockk.mockk import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Disabled @@ -65,6 +67,7 @@ class MutexControlTest { buildId = buildId, stageId = stageId, containerId = containerId, + jobId = "job-123", containerHashId = containerHashId, containerType = "vmBuild", seq = containerId.toInt(), @@ -73,17 +76,20 @@ class MutexControlTest { matrixGroupId = null, matrixGroupFlag = false ) + private val engineConfigMock = mockk() private val mutexControl: MutexControl = MutexControl( buildLogPrinter = buildLogPrinter, redisOperation = redisOperation, containerBuildRecordService = mockk(), pipelineUrlBean = mockk(), - pipelineContainerService = mockk() + pipelineContainerService = mockk(), + engineConfigService = engineConfigMock ) @Test // 测试MutexControl的初始化功能 fun initMutexGroup() { + every { engineConfigMock.getMutexMaxQueue() } returns 10 val initMutexGroup = mutexControl.decorateMutexGroup( mutexGroup = mutexGroup, variables = variables diff --git a/src/backend/ci/core/process/biz-engine/src/test/kotlin/com/tencent/devops/process/engine/utils/TestTool.kt b/src/backend/ci/core/process/biz-engine/src/test/kotlin/com/tencent/devops/process/engine/utils/TestTool.kt index 15857d4ed1d..ad8391be2bb 100644 --- a/src/backend/ci/core/process/biz-engine/src/test/kotlin/com/tencent/devops/process/engine/utils/TestTool.kt +++ b/src/backend/ci/core/process/biz-engine/src/test/kotlin/com/tencent/devops/process/engine/utils/TestTool.kt @@ -64,6 +64,7 @@ object TestTool { containerId = vmSeqId.toString(), seq = vmSeqId, containerHashId = containerHashId, + jobId = "job-123", containerType = vmContainerType, status = status, startTime = startTime, diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/ServiceBuildResourceImpl.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/ServiceBuildResourceImpl.kt index fff1f26ac91..4502e8b1f50 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/ServiceBuildResourceImpl.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/ServiceBuildResourceImpl.kt @@ -385,7 +385,9 @@ class ServiceBuildResourceImpl @Autowired constructor( buildMsg: String?, startUser: List?, archiveFlag: Boolean?, - debugVersion: Int? + customVersion: Int?, + triggerAlias: List?, + triggerBranch: List? ): Result> { checkUserId(userId) checkParam(projectId, pipelineId) @@ -419,7 +421,9 @@ class ServiceBuildResourceImpl @Autowired constructor( startUser = startUser?.filter { it.isNotBlank() }, updateTimeDesc = updateTimeDesc, archiveFlag = archiveFlag, - debugVersion = debugVersion + customVersion = customVersion, + triggerAlias = triggerAlias, + triggerBranch = triggerBranch ) return Result(result) } @@ -636,8 +640,9 @@ class ServiceBuildResourceImpl @Autowired constructor( channelCode: ChannelCode? ): Result { val history = pipelineBuildFacadeService.getSingleHistoryBuild( - projectId, pipelineId, - buildNum.toInt(), channelCode ?: ChannelCode.BS + projectId = projectId, pipelineId = pipelineId, + buildNum = buildNum.toInt(), buildId = null, + channelCode = channelCode ?: ChannelCode.BS ) return Result(history) } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/ServicePipelineResourceImpl.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/ServicePipelineResourceImpl.kt index 87485ac400e..16dd5718b9a 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/ServicePipelineResourceImpl.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/ServicePipelineResourceImpl.kt @@ -45,6 +45,7 @@ import com.tencent.devops.common.pipeline.Model import com.tencent.devops.common.pipeline.ModelUpdate import com.tencent.devops.common.pipeline.enums.ChannelCode import com.tencent.devops.common.pipeline.pojo.PipelineModelAndSetting +import com.tencent.devops.common.pipeline.pojo.setting.PipelineSetting import com.tencent.devops.common.web.RestResource import com.tencent.devops.common.web.utils.I18nUtil import com.tencent.devops.process.api.service.ServicePipelineResource @@ -53,6 +54,7 @@ import com.tencent.devops.process.engine.pojo.PipelineInfo import com.tencent.devops.process.engine.service.PipelineRepositoryService import com.tencent.devops.process.engine.service.PipelineRepositoryService.Companion.checkParam import com.tencent.devops.process.engine.service.rule.PipelineRuleService +import com.tencent.devops.process.enums.OperationLogType import com.tencent.devops.process.permission.PipelinePermissionService import com.tencent.devops.process.pojo.Permission import com.tencent.devops.process.pojo.Pipeline @@ -61,15 +63,13 @@ import com.tencent.devops.process.pojo.PipelineId import com.tencent.devops.process.pojo.PipelineIdAndName import com.tencent.devops.process.pojo.PipelineIdInfo import com.tencent.devops.process.pojo.PipelineName +import com.tencent.devops.process.pojo.PipelineRemoteToken import com.tencent.devops.process.pojo.PipelineSortType import com.tencent.devops.process.pojo.audit.Audit import com.tencent.devops.process.pojo.classify.PipelineViewPipelinePage import com.tencent.devops.process.pojo.pipeline.DeployPipelineResult import com.tencent.devops.process.pojo.pipeline.SimplePipeline import com.tencent.devops.process.pojo.pipeline.enums.PipelineRuleBusCodeEnum -import com.tencent.devops.common.pipeline.pojo.setting.PipelineSetting -import com.tencent.devops.process.enums.OperationLogType -import com.tencent.devops.process.pojo.PipelineRemoteToken import com.tencent.devops.process.service.PipelineInfoFacadeService import com.tencent.devops.process.service.PipelineListFacadeService import com.tencent.devops.process.service.PipelineRemoteAuthService @@ -400,7 +400,8 @@ class ServicePipelineResourceImpl @Autowired constructor( userId: String, projectId: String, pipelineId: String, - channelCode: ChannelCode + channelCode: ChannelCode, + checkFlag: Boolean? ): Result { checkParams(userId, projectId) pipelineInfoFacadeService.deletePipeline( @@ -408,7 +409,7 @@ class ServicePipelineResourceImpl @Autowired constructor( projectId = projectId, pipelineId = pipelineId, channelCode = channelCode, - checkPermission = ChannelCode.isNeedAuth(channelCode) + checkPermission = ChannelCode.isNeedAuth(channelCode) && checkFlag == true ) return Result(true) } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/ServicePipelineVersionResourceImpl.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/ServicePipelineVersionResourceImpl.kt index f8969f37f08..a328b4b778b 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/ServicePipelineVersionResourceImpl.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/ServicePipelineVersionResourceImpl.kt @@ -162,7 +162,7 @@ class ServicePipelineVersionResourceImpl @Autowired constructor( AuthPermission.CREATE ) return Result( - pipelineVersionFacadeService.createPipelineFromTemplate( + pipelineVersionFacadeService.createPipelineFromFreedom( userId = userId, projectId = projectId, request = request diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserBuildResourceImpl.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserBuildResourceImpl.kt index 4f30ca56ca6..8c4585739c7 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserBuildResourceImpl.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserBuildResourceImpl.kt @@ -42,6 +42,7 @@ import com.tencent.devops.common.pipeline.pojo.element.Element import com.tencent.devops.common.web.RestResource import com.tencent.devops.process.api.user.UserBuildResource import com.tencent.devops.process.engine.service.PipelineProgressRateService +import com.tencent.devops.process.enums.HistorySearchType import com.tencent.devops.process.pojo.BuildHistory import com.tencent.devops.process.pojo.BuildHistoryRemark import com.tencent.devops.process.pojo.BuildId @@ -450,7 +451,9 @@ class UserBuildResourceImpl @Autowired constructor( buildNoEnd: Int?, buildMsg: String?, archiveFlag: Boolean?, - debugVersion: Int? + customVersion: Int?, + triggerAlias: List?, + triggerBranch: List? ): Result> { checkParam(userId, projectId, pipelineId) val result = pipelineBuildFacadeService.getHistoryBuild( @@ -479,7 +482,9 @@ class UserBuildResourceImpl @Autowired constructor( buildNoEnd = buildNoEnd, buildMsg = buildMsg, archiveFlag = archiveFlag, - debugVersion = debugVersion + customVersion = customVersion, + triggerAlias = triggerAlias, + triggerBranch = triggerBranch ) if (archiveFlag != true) { pipelineRecentUseService.record(userId, projectId, pipelineId) @@ -527,12 +532,21 @@ class UserBuildResourceImpl @Autowired constructor( userId: String, projectId: String, pipelineId: String, - debugVersion: Int? + debugVersion: Int?, + search: String?, + type: HistorySearchType? ): Result> { checkParam(userId, projectId, pipelineId) - return Result(pipelineBuildFacadeService.getHistoryConditionRepo( - userId, projectId, pipelineId, debugVersion - )) + return Result( + pipelineBuildFacadeService.getHistoryConditionRepo( + userId = userId, + projectId = projectId, + pipelineId = pipelineId, + debugVersion = debugVersion, + search = search, + type = type + ) + ) } override fun getHistoryConditionBranch( @@ -540,7 +554,9 @@ class UserBuildResourceImpl @Autowired constructor( projectId: String, pipelineId: String, alias: List?, - debugVersion: Int? + debugVersion: Int?, + search: String?, + type: HistorySearchType? ): Result> { checkParam(userId, projectId, pipelineId) return Result( @@ -549,7 +565,9 @@ class UserBuildResourceImpl @Autowired constructor( projectId = projectId, pipelineId = pipelineId, alias = alias, - debugVersion = debugVersion + debugVersion = debugVersion, + search = search, + type = type ) ) } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserPipelineResourceImpl.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserPipelineResourceImpl.kt index 83f2e6ba1bf..196a5097fa5 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserPipelineResourceImpl.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserPipelineResourceImpl.kt @@ -45,12 +45,10 @@ import com.tencent.devops.common.pipeline.enums.ChannelCode import com.tencent.devops.common.pipeline.enums.VersionStatus import com.tencent.devops.common.pipeline.pojo.MatrixPipelineInfo import com.tencent.devops.common.pipeline.pojo.PipelineModelAndSetting -import com.tencent.devops.common.pipeline.pojo.setting.PipelineRunLockType import com.tencent.devops.common.pipeline.pojo.setting.PipelineSetting import com.tencent.devops.common.pipeline.utils.MatrixYamlCheckUtils import com.tencent.devops.common.web.RestResource import com.tencent.devops.common.web.utils.I18nUtil -import com.tencent.devops.process.api.service.ServicePipelineResource import com.tencent.devops.process.api.user.UserPipelineResource import com.tencent.devops.process.audit.service.AuditService import com.tencent.devops.process.constant.ProcessMessageCode @@ -136,14 +134,22 @@ class UserPipelineResourceImpl @Autowired constructor( pageSize: Int? ): Result> { checkParam(userId, projectId) - // TODO 权限迁移完后应该删除掉 - return client.getGateway(ServicePipelineResource::class).hasPermissionList( + val result = pipelineListFacadeService.hasPermissionList( userId = userId, projectId = projectId, permission = permission, excludePipelineId = excludePipelineId, + filterByPipelineName = null, page = page, - pageSize + pageSize = pageSize + ) + return Result( + data = Page( + page = page ?: 0, + pageSize = pageSize ?: -1, + count = result.count, + records = result.records + ) ) } @@ -352,38 +358,21 @@ class UserPipelineResourceImpl @Autowired constructor( enable: Boolean ): Result { checkParam(userId, projectId) - val origin = pipelineSettingFacadeService.userGetSetting(userId, projectId, pipelineId) - // 暂时无法回溯关闭前的配置，先采用支持并发的配置 - val operationLogType: OperationLogType - val setting = if (enable) { - operationLogType = OperationLogType.ENABLE_PIPELINE - origin.copy(runLockType = PipelineRunLockType.MULTIPLE) - } else { - operationLogType = OperationLogType.DISABLE_PIPELINE - origin.copy(runLockType = PipelineRunLockType.LOCK) - } - val savedSetting = pipelineSettingFacadeService.saveSetting( + + val pipelineInfo = pipelineInfoFacadeService.locked( userId = userId, projectId = projectId, pipelineId = pipelineId, - setting = setting, - checkPermission = true - ) - pipelineInfoFacadeService.updatePipelineSettingVersion( - userId = userId, - projectId = setting.projectId, - pipelineId = setting.pipelineId, - operationLogType = operationLogType, - savedSetting = savedSetting + locked = !enable ) auditService.createAudit( Audit( resourceType = AuthResourceType.PIPELINE_DEFAULT.value, resourceId = pipelineId, - resourceName = setting.pipelineName, + resourceName = pipelineInfo.pipelineName, userId = userId, action = "edit", - actionContent = "Update Setting", + actionContent = if (enable) "UnLock Pipeline" else "Locked Pipeline", projectId = projectId ) ) @@ -632,7 +621,8 @@ class UserPipelineResourceImpl @Autowired constructor( filterByLabels = filterByLabels, filterByViewIds = filterByViewIds, collation = collation ?: PipelineCollation.DEFAULT, - showDelete = showDelete ?: false + showDelete = showDelete ?: false, + queryByWeb = true ) ) } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserPipelineVersionResourceImpl.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserPipelineVersionResourceImpl.kt index d5e890f244e..da28ce80ca8 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserPipelineVersionResourceImpl.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserPipelineVersionResourceImpl.kt @@ -43,7 +43,6 @@ import com.tencent.devops.common.web.RestResource import com.tencent.devops.common.web.utils.I18nUtil import com.tencent.devops.process.api.user.UserPipelineVersionResource import com.tencent.devops.process.audit.service.AuditService -import com.tencent.devops.process.engine.pojo.PipelineVersionWithInfo import com.tencent.devops.process.permission.PipelinePermissionService import com.tencent.devops.process.pojo.PipelineDetail import com.tencent.devops.process.pojo.PipelineOperationDetail @@ -163,7 +162,7 @@ class UserPipelineVersionResourceImpl @Autowired constructor( AuthPermission.CREATE ) return Result( - pipelineVersionFacadeService.createPipelineFromTemplate( + pipelineVersionFacadeService.createPipelineFromFreedom( userId = userId, projectId = projectId, request = request @@ -302,7 +301,7 @@ class UserPipelineVersionResourceImpl @Autowired constructor( description: String?, page: Int?, pageSize: Int? - ): Result> { + ): Result> { checkParam(userId, projectId) val permission = AuthPermission.VIEW pipelinePermissionService.validPipelinePermission( @@ -322,7 +321,7 @@ class UserPipelineVersionResourceImpl @Autowired constructor( ) ) return Result( - pipelineVersionFacadeService.listPipelineVersionInfo( + pipelineVersionFacadeService.listPipelineVersion( projectId = projectId, pipelineId = pipelineId, fromVersion = fromVersion, @@ -341,7 +340,7 @@ class UserPipelineVersionResourceImpl @Autowired constructor( projectId: String, pipelineId: String, version: Int - ): Result { + ): Result { checkParam(userId, projectId) val permission = AuthPermission.VIEW pipelinePermissionService.validPipelinePermission( @@ -361,7 +360,7 @@ class UserPipelineVersionResourceImpl @Autowired constructor( ) ) return Result( - pipelineVersionFacadeService.getPipelineVersionInfo( + pipelineVersionFacadeService.getPipelineVersion( projectId = projectId, pipelineId = pipelineId, version = version diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserSubPipelineInfoResourceImpl.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserSubPipelineInfoResourceImpl.kt index 6bfecf940eb..a9966344a4b 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserSubPipelineInfoResourceImpl.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/UserSubPipelineInfoResourceImpl.kt @@ -29,32 +29,34 @@ package com.tencent.devops.process.api import com.tencent.devops.common.api.exception.ParamBlankException import com.tencent.devops.common.api.pojo.Result -import com.tencent.devops.common.client.Client import com.tencent.devops.common.web.RestResource -import com.tencent.devops.process.api.service.ServiceSubPipelineResource import com.tencent.devops.process.api.user.UserSubPipelineInfoResource import com.tencent.devops.process.pojo.pipeline.SubPipelineStartUpInfo +import com.tencent.devops.process.service.SubPipelineStartUpService import org.springframework.beans.factory.annotation.Autowired @RestResource class UserSubPipelineInfoResourceImpl @Autowired constructor ( - private val client: Client + private val subPipeService: SubPipelineStartUpService ) : UserSubPipelineInfoResource { override fun subpipManualStartupInfo( userId: String, projectId: String, - pipelineId: String + pipelineId: String, + includeConst: Boolean?, + includeNotRequired: Boolean? ): Result> { checkParam(userId) if (pipelineId.isBlank() || projectId.isBlank()) { return Result(ArrayList()) } - // TODO 权限迁移完后应该删除掉 - return client.getGateway(ServiceSubPipelineResource::class).subpipManualStartupInfo( + return subPipeService.subPipelineManualStartupInfo( userId = userId, projectId = projectId, - pipelineId = pipelineId + pipelineId = pipelineId, + includeConst = includeConst, + includeNotRequired = includeNotRequired ) } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/app/AppPipelineBuildResourceImpl.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/app/AppPipelineBuildResourceImpl.kt index 6526ecaf0b6..55ff59ccf21 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/app/AppPipelineBuildResourceImpl.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/app/AppPipelineBuildResourceImpl.kt @@ -198,9 +198,11 @@ class AppPipelineBuildResourceImpl @Autowired constructor( buildNoStart: Int?, buildNoEnd: Int?, buildMsg: String?, - debugVersion: Int? + customVersion: Int?, + triggerAlias: List?, + triggerBranch: List? ): Result> { - checkParam(userId, projectId, pipelineId) + checkParam(userId, projectId, pipelineId, pageSize) val result = pipelineBuildFacadeService.getHistoryBuild( userId = userId, projectId = projectId, @@ -226,7 +228,9 @@ class AppPipelineBuildResourceImpl @Autowired constructor( buildNoStart = buildNoStart, buildNoEnd = buildNoEnd, buildMsg = buildMsg, - debugVersion = debugVersion + customVersion = customVersion, + triggerAlias = triggerAlias, + triggerBranch = triggerBranch ) return Result(result) } @@ -361,7 +365,7 @@ class AppPipelineBuildResourceImpl @Autowired constructor( ) } - private fun checkParam(userId: String, projectId: String, pipelineId: String) { + private fun checkParam(userId: String, projectId: String, pipelineId: String, pageSize: Int? = null) { if (userId.isBlank()) { throw ParamBlankException("Invalid userId") } @@ -371,5 +375,8 @@ class AppPipelineBuildResourceImpl @Autowired constructor( if (projectId.isBlank()) { throw ParamBlankException("Invalid projectId") } + if (pageSize != null && pageSize > 1000) { + throw ParamBlankException("PageSize could not be greater than 1000") + } } } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/builds/BuildBuildResourceImpl.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/builds/BuildBuildResourceImpl.kt index 52c904e4175..904977834f0 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/builds/BuildBuildResourceImpl.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/builds/BuildBuildResourceImpl.kt @@ -52,6 +52,7 @@ class BuildBuildResourceImpl @Autowired constructor( projectId: String, pipelineId: String, buildNum: String, + buildId: String?, channelCode: ChannelCode? ): Result { return Result( @@ -59,6 +60,7 @@ class BuildBuildResourceImpl @Autowired constructor( projectId = projectId, pipelineId = pipelineId, buildNum = buildNum.toInt(), + buildId = buildId, channelCode = channelCode ?: ChannelCode.BS ) ) @@ -68,12 +70,14 @@ class BuildBuildResourceImpl @Autowired constructor( override fun getLatestSuccessBuild( projectId: String, pipelineId: String, + buildId: String?, channelCode: ChannelCode? ): Result { return Result( data = pipelineBuildFacadeService.getLatestSuccessBuild( projectId = projectId, pipelineId = pipelineId, + buildId = buildId, channelCode = channelCode ?: ChannelCode.BS ) ) diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/builds/BuildSubPipelineResourceImpl.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/builds/BuildSubPipelineResourceImpl.kt index 8da6e222955..29bebc7e8ab 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/builds/BuildSubPipelineResourceImpl.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/builds/BuildSubPipelineResourceImpl.kt @@ -31,10 +31,8 @@ import com.tencent.bk.audit.annotations.AuditEntry import com.tencent.devops.common.api.exception.ParamBlankException import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.auth.api.ActionId -import com.tencent.devops.common.client.Client import com.tencent.devops.common.pipeline.enums.ChannelCode import com.tencent.devops.common.web.RestResource -import com.tencent.devops.process.api.service.ServiceSubPipelineResource import com.tencent.devops.process.pojo.PipelineId import com.tencent.devops.process.pojo.pipeline.ProjectBuildId import com.tencent.devops.process.pojo.pipeline.SubPipelineStartUpInfo @@ -44,8 +42,7 @@ import org.springframework.beans.factory.annotation.Autowired @RestResource class BuildSubPipelineResourceImpl @Autowired constructor( - private val subPipeService: SubPipelineStartUpService, - private val client: Client + private val subPipeService: SubPipelineStartUpService ) : BuildSubPipelineResource { @AuditEntry(actionId = ActionId.PIPELINE_EXECUTE) @@ -58,34 +55,21 @@ class BuildSubPipelineResourceImpl @Autowired constructor( atomCode: String, taskId: String, runMode: String, - values: Map + values: Map, + executeCount: Int? ): Result { - return if (projectId != callProjectId) { - // TODO 权限迁移完后应该删除掉 - client.getGateway(ServiceSubPipelineResource::class).callOtherProjectPipelineStartup( - callProjectId = callProjectId, - callPipelineId = callPipelineId, - atomCode = atomCode, - parentProjectId = projectId, - parentPipelineId = parentPipelineId, - buildId = buildId, - taskId = taskId, - runMode = runMode, - values = values - ) - } else { - subPipeService.callPipelineStartup( - projectId = projectId, - parentPipelineId = parentPipelineId, - buildId = buildId, - callProjectId = callProjectId, - callPipelineId = callPipelineId, - atomCode = atomCode, - taskId = taskId, - runMode = runMode, - values = values - ) - } + return subPipeService.callPipelineStartup( + projectId = projectId, + parentPipelineId = parentPipelineId, + buildId = buildId, + callProjectId = callProjectId, + callPipelineId = callPipelineId, + atomCode = atomCode, + taskId = taskId, + runMode = runMode, + values = values, + executeCount = executeCount + ) } override fun callPipelineStartup( @@ -97,7 +81,8 @@ class BuildSubPipelineResourceImpl @Autowired constructor( taskId: String, runMode: String, channelCode: ChannelCode?, - values: Map + values: Map, + executeCount: Int? ): Result { return subPipeService.callPipelineStartup( projectId = projectId, @@ -108,7 +93,8 @@ class BuildSubPipelineResourceImpl @Autowired constructor( taskId = taskId, runMode = runMode, channelCode = channelCode, - values = values + values = values, + executeCount = executeCount ) } @@ -123,13 +109,17 @@ class BuildSubPipelineResourceImpl @Autowired constructor( override fun subpipManualStartupInfo( userId: String, projectId: String, - pipelineId: String + pipelineId: String, + includeConst: Boolean?, + includeNotRequired: Boolean? ): Result> { checkParam(userId) - return client.getGateway(ServiceSubPipelineResource::class).subpipManualStartupInfo( + return subPipeService.subPipelineManualStartupInfo( userId = userId, projectId = projectId, - pipelineId = pipelineId + pipelineId = pipelineId, + includeConst = includeConst, + includeNotRequired = includeNotRequired ) } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/external/ExternalScmResourceImpl.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/external/ExternalScmResourceImpl.kt index 3f47a797617..9bbbc3cdc30 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/external/ExternalScmResourceImpl.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/external/ExternalScmResourceImpl.kt @@ -58,18 +58,25 @@ class ExternalScmResourceImpl @Autowired constructor( override fun webHookCodeGitCommit( event: String, secret: String?, + sourceType: String?, traceId: String, body: String - ): Result = Result( - CodeWebhookEventDispatcher.dispatchEvent( - rabbitTemplate = rabbitTemplate, - event = GitWebhookEvent( - requestContent = body, - event = event, - secret = secret + ): Result { + // 工蜂的测试请求,应该忽略 + if (sourceType == "Test") { + return Result(true) + } + return Result( + CodeWebhookEventDispatcher.dispatchEvent( + rabbitTemplate = rabbitTemplate, + event = GitWebhookEvent( + requestContent = body, + event = event, + secret = secret + ) ) ) - ) + } override fun webHookGitlabCommit(event: String) = Result(CodeWebhookEventDispatcher.dispatchEvent(rabbitTemplate, GitlabWebhookEvent(requestContent = event))) diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/op/OpPipelineSettingResourceImpl.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/op/OpPipelineSettingResourceImpl.kt index 43ed892cdb4..2f118578577 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/op/OpPipelineSettingResourceImpl.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/op/OpPipelineSettingResourceImpl.kt @@ -35,12 +35,12 @@ import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.api.util.MessageUtil import com.tencent.devops.common.auth.api.ActionId import com.tencent.devops.common.client.Client +import com.tencent.devops.common.pipeline.pojo.setting.PipelineSetting import com.tencent.devops.common.web.RestResource import com.tencent.devops.common.web.utils.I18nUtil import com.tencent.devops.process.constant.ProcessMessageCode.MAXIMUM_NUMBER_CONCURRENCY_ILLEGAL import com.tencent.devops.process.constant.ProcessMessageCode.PROJECT_NOT_EXIST import com.tencent.devops.process.dao.PipelineSettingDao -import com.tencent.devops.common.pipeline.pojo.setting.PipelineSetting import com.tencent.devops.process.service.pipeline.PipelineSettingFacadeService import com.tencent.devops.process.utils.PIPELINE_SETTING_MAX_CON_QUEUE_SIZE_MAX import com.tencent.devops.process.utils.PIPELINE_SETTING_MAX_QUEUE_SIZE_MIN @@ -134,6 +134,24 @@ class OpPipelineSettingResourceImpl @Autowired constructor( ) } + override fun updateBuildMetricsSettings(userId: String, projectId: String, enabled: Boolean): Result { + logger.info( + "[$projectId]|updateBuildMetricsSettings|userId=$userId|$enabled" + ) + val projectVO = client.get(ServiceProjectResource::class).get(projectId).data + ?: throw ExecuteException( + MessageUtil.getMessageByLocale(PROJECT_NOT_EXIST, I18nUtil.getLanguage(userId)) + ) + val success = client.get(OPProjectResource::class).setProjectProperties( + userId = userId, + projectCode = projectId, + properties = projectVO.properties?.copy( + buildMetrics = enabled + ) ?: ProjectProperties(buildMetrics = enabled) + ).data == true + return Result(success) + } + private fun checkMaxConRunningQueueSize(maxConRunningQueueSize: Int) { if (maxConRunningQueueSize <= PIPELINE_SETTING_MAX_QUEUE_SIZE_MIN || maxConRunningQueueSize > PIPELINE_SETTING_MAX_CON_QUEUE_SIZE_MAX diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/open/OpenPipelineTaskResourceImpl.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/open/OpenPipelineTaskResourceImpl.kt new file mode 100644 index 00000000000..f336848739b --- /dev/null +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/open/OpenPipelineTaskResourceImpl.kt @@ -0,0 +1,43 @@ +package com.tencent.devops.process.api.open + +import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.common.web.RestResource +import com.tencent.devops.process.engine.service.PipelineRuntimeService +import com.tencent.devops.process.engine.service.PipelineTaskService +import com.tencent.devops.process.pojo.open.BuildStatusInfo +import org.springframework.beans.factory.annotation.Autowired + +@RestResource +class OpenPipelineTaskResourceImpl @Autowired constructor( + private val pipelineTaskService: PipelineTaskService, + private val pipelineRuntimeService: PipelineRuntimeService +) : OpenPipelineTaskResource { + + override fun getBuildStatus( + projectId: String, + pipelineId: String, + buildId: String, + taskId: String? + ): Result { + + // 校验参数 + val build = pipelineRuntimeService.getBuildInfo( + projectId = projectId, + pipelineId = pipelineId, + buildId = buildId + ) + + return if (build == null) { + Result(status = -1, message = "Build[$buildId] is not found!") + } else if (!taskId.isNullOrBlank()) { // 查指定task的状态 + val tStatus = pipelineTaskService.getTaskStatus(projectId = projectId, buildId = buildId, taskId = taskId) + if (tStatus == null) { + Result(status = -1, message = "Task[$taskId] is not found!") + } else { + Result(BuildStatusInfo(startUser = build.startUser, debug = build.debug, status = tStatus)) + } + } else { + Result(BuildStatusInfo(startUser = build.startUser, debug = build.debug, status = build.status)) + } + } +} diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/service/ServiceSubPipelineResourceImpl.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/service/ServiceSubPipelineResourceImpl.kt index b396a6c1573..68c28ca2e51 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/service/ServiceSubPipelineResourceImpl.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/api/service/ServiceSubPipelineResourceImpl.kt @@ -44,7 +44,7 @@ class ServiceSubPipelineResourceImpl constructor( projectId: String, pipelineId: String ): Result> { - return subPipeService.subPipelineManualStartupInfo(userId, projectId, pipelineId) + return subPipeService.subPipelineManualStartupInfo(userId, projectId, pipelineId, true, true) } override fun callOtherProjectPipelineStartup( @@ -54,6 +54,7 @@ class ServiceSubPipelineResourceImpl constructor( parentProjectId: String, parentPipelineId: String, buildId: String, + executeCount: Int?, taskId: String, runMode: String, values: Map @@ -67,7 +68,8 @@ class ServiceSubPipelineResourceImpl constructor( atomCode = atomCode, taskId = taskId, runMode = runMode, - values = values + values = values, + executeCount = executeCount ) } } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/dao/PipelineTriggerEventDao.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/dao/PipelineTriggerEventDao.kt index 090165d2931..9ea38ff7885 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/dao/PipelineTriggerEventDao.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/dao/PipelineTriggerEventDao.kt @@ -126,7 +126,7 @@ class PipelineTriggerEventDao { triggerDetail.reason, triggerDetail.reasonDetail?.let { JsonUtil.toJson(it, true) }, LocalDateTime.now() - ).execute() + ).onDuplicateKeyIgnore().execute() } } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/engine/control/CallBackControl.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/engine/control/CallBackControl.kt index 423f9607272..adee440ae74 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/engine/control/CallBackControl.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/engine/control/CallBackControl.kt @@ -32,7 +32,6 @@ import com.tencent.devops.common.api.exception.RemoteServiceException import com.tencent.devops.common.api.util.Watcher import com.tencent.devops.common.api.util.timestampmilli import com.tencent.devops.common.client.Client -import com.tencent.devops.common.event.enums.ActionType import com.tencent.devops.common.event.pojo.pipeline.PipelineBuildStatusBroadCastEvent import com.tencent.devops.common.pipeline.Model import com.tencent.devops.common.pipeline.container.Container @@ -51,6 +50,7 @@ import com.tencent.devops.common.pipeline.event.SimpleModel import com.tencent.devops.common.pipeline.event.SimpleStage import com.tencent.devops.common.pipeline.event.SimpleTask import com.tencent.devops.common.pipeline.event.StreamEnabledEvent +import com.tencent.devops.common.pipeline.utils.EventUtils.toEventType import com.tencent.devops.common.service.trace.TraceTag import com.tencent.devops.common.service.utils.LogUtils import com.tencent.devops.common.util.HttpRetryUtils @@ -212,31 +212,10 @@ class CallBackControl @Autowired constructor( val projectId = event.projectId val pipelineId = event.pipelineId val buildId = event.buildId - - val callBackEvent = - if (event.taskId.isNullOrBlank()) { - if (event.stageId.isNullOrBlank()) { - if (event.actionType == ActionType.START) { - CallBackEvent.BUILD_START - } else { - CallBackEvent.BUILD_END - } - } else { - if (event.actionType == ActionType.START) { - CallBackEvent.BUILD_STAGE_START - } else { - CallBackEvent.BUILD_STAGE_END - } - } - } else { - if (event.actionType == ActionType.START) { - CallBackEvent.BUILD_TASK_START - } else if (event.actionType == ActionType.REFRESH) { - CallBackEvent.BUILD_TASK_PAUSE - } else { - CallBackEvent.BUILD_TASK_END - } - } + if (event.atomCode != null && VmOperateTaskGenerator.isVmAtom(event.atomCode!!)) { + return + } + val callBackEvent = event.toEventType() ?: return logger.info("$projectId|$pipelineId|$buildId|${callBackEvent.name}|${event.stageId}|${event.taskId}|callback") val list = mutableListOf() @@ -292,9 +271,11 @@ class CallBackControl @Autowired constructor( is PipelineEvent -> { data.pipelineId } + is BuildEvent -> { data.buildId } + else -> "" } val watcher = Watcher(id = "${it.projectId}|${it.callBackUrl}|${it.events}|$uniqueId") diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/notify/PipelineBuildNotifyListener.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/notify/PipelineBuildNotifyListener.kt index 0e30d79f675..2b7895209b3 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/notify/PipelineBuildNotifyListener.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/notify/PipelineBuildNotifyListener.kt @@ -34,6 +34,7 @@ import com.tencent.devops.common.event.listener.pipeline.BaseListener import com.tencent.devops.common.web.utils.I18nUtil import com.tencent.devops.notify.api.service.ServiceNotifyMessageTemplateResource import com.tencent.devops.notify.pojo.SendNotifyMessageTemplateRequest +import com.tencent.devops.process.api.service.ServicePipelineResource import com.tencent.devops.process.bean.PipelineUrlBean import com.tencent.devops.process.constant.ProcessMessageCode.BK_BUILD_IN_REVIEW_STATUS import com.tencent.devops.process.engine.pojo.event.PipelineBuildNotifyEvent @@ -51,6 +52,16 @@ class PipelineBuildNotifyListener @Autowired constructor( ) : BaseListener(pipelineEventDispatcher) { override fun run(event: PipelineBuildNotifyEvent) { + try { + val pipelineNotDeleted = client.get(ServicePipelineResource::class) + .getPipelineInfo(projectId = event.projectId, pipelineId = event.pipelineId, channelCode = null).data + if (pipelineNotDeleted == null) { + logger.warn("NOTIFY|CHECK_PIPE|Pipeline[${event.projectId}/${event.pipelineId}] may be deleted!") + return + } + } catch (ignore: Exception) { + logger.warn("NOTIFY|CHECK_PIPE|SKIP_ERROR_CHECK", ignore) + } when (val notifyTemplateEnumType = PipelineNotifyTemplateEnum.parse(event.notifyTemplateEnum)) { PipelineNotifyTemplateEnum.PIPELINE_MANUAL_REVIEW_STAGE_NOTIFY_TEMPLATE, PipelineNotifyTemplateEnum.PIPELINE_MANUAL_REVIEW_ATOM_NOTIFY_TEMPLATE, @@ -77,9 +88,11 @@ class PipelineBuildNotifyListener @Autowired constructor( ) } } + PipelineNotifyTemplateEnum.PIPELINE_MANUAL_REVIEW_ATOM_REMINDER_NOTIFY_TEMPLATE -> { event.sendReviewReminder() } + else -> { // need to add } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/notify/command/impl/BluekingNotifySendCmd.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/notify/command/impl/BluekingNotifySendCmd.kt index 2ad4584dba1..1922a05db8a 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/notify/command/impl/BluekingNotifySendCmd.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/notify/command/impl/BluekingNotifySendCmd.kt @@ -3,6 +3,8 @@ package com.tencent.devops.process.notify.command.impl import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.api.util.EnvUtils import com.tencent.devops.common.client.Client +import com.tencent.devops.common.notify.utils.NotifyUtils +import com.tencent.devops.common.pipeline.pojo.setting.PipelineSubscriptionType import com.tencent.devops.process.notify.command.BuildNotifyContext import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service @@ -31,8 +33,14 @@ class BluekingNotifySendCmd @Autowired constructor( val successContent = EnvUtils.parseEnv( successSubscription.content, commandContext.variables, replaceWithEmpty ) + val group = EnvUtils.parseEnv( + command = successSubscription.wechatGroup, + data = commandContext.variables, + replaceWithEmpty = true + ) commandContext.notifyValue["successContent"] = successContent commandContext.notifyValue["emailSuccessContent"] = successContent + commandContext.notifyValue[NotifyUtils.WEWORK_GROUP_KEY] = group val receivers = successSubscription.users.split(",").map { EnvUtils.parseEnv( command = it, @@ -43,10 +51,24 @@ class BluekingNotifySendCmd @Autowired constructor( sendNotifyByTemplate( templateCode = getNotifyTemplateCode(shutdownType, successSubscription.detailFlag), receivers = receivers, - notifyType = successSubscription.types.map { it.name }.toMutableSet(), + notifyType = successSubscription.types.filter { + it != PipelineSubscriptionType.WEWORK_GROUP + }.map { it.name }.toMutableSet(), titleParams = commandContext.notifyValue, - bodyParams = commandContext.notifyValue + bodyParams = commandContext.notifyValue, + markdownContent = false ) + // 企业微信通知组的模板和企业微信通知用的是同一个模板,但是企业微信通知没有markdown选项,所以需要单独发送 + if (successSubscription.types.contains(PipelineSubscriptionType.WEWORK_GROUP)) { + sendNotifyByTemplate( + templateCode = getNotifyTemplateCode(shutdownType, successSubscription.detailFlag), + receivers = receivers, + notifyType = setOf(PipelineSubscriptionType.WEWORK_GROUP.name), + titleParams = commandContext.notifyValue, + bodyParams = commandContext.notifyValue, + markdownContent = successSubscription.wechatGroupMarkdownFlag + ) + } } } buildStatus.isFailure() -> { @@ -55,8 +77,14 @@ class BluekingNotifySendCmd @Autowired constructor( val failContent = EnvUtils.parseEnv( failSubscription.content, commandContext.variables, replaceWithEmpty ) + val group = EnvUtils.parseEnv( + command = failSubscription.wechatGroup, + data = commandContext.variables, + replaceWithEmpty = true + ) commandContext.notifyValue["failContent"] = failContent commandContext.notifyValue["emailFailContent"] = failContent + commandContext.notifyValue[NotifyUtils.WEWORK_GROUP_KEY] = group val receivers = failSubscription.users.split(",").map { EnvUtils.parseEnv( command = it, @@ -67,10 +95,24 @@ class BluekingNotifySendCmd @Autowired constructor( sendNotifyByTemplate( templateCode = getNotifyTemplateCode(shutdownType, failSubscription.detailFlag), receivers = receivers, - notifyType = failSubscription.types.map { it.name }.toMutableSet(), + notifyType = failSubscription.types.filter { + it != PipelineSubscriptionType.WEWORK_GROUP + }.map { it.name }.toMutableSet(), titleParams = commandContext.notifyValue, - bodyParams = commandContext.notifyValue + bodyParams = commandContext.notifyValue, + markdownContent = false ) + // 企业微信通知组的模板和企业微信通知用的是同一个模板,但是企业微信通知没有markdown选项,所以需要单独发送 + if (failSubscription.types.contains(PipelineSubscriptionType.WEWORK_GROUP)) { + sendNotifyByTemplate( + templateCode = getNotifyTemplateCode(shutdownType, failSubscription.detailFlag), + receivers = receivers, + notifyType = setOf(PipelineSubscriptionType.WEWORK_GROUP.name), + titleParams = commandContext.notifyValue, + bodyParams = commandContext.notifyValue, + markdownContent = failSubscription.wechatGroupMarkdownFlag + ) + } } } else -> Result(0) diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/notify/command/impl/NotifySendCmd.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/notify/command/impl/NotifySendCmd.kt index 64047f69846..9327e214319 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/notify/command/impl/NotifySendCmd.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/notify/command/impl/NotifySendCmd.kt @@ -13,7 +13,8 @@ abstract class NotifySendCmd(val client: Client) : NotifyCmd { receivers: Set, notifyType: Set, titleParams: Map, - bodyParams: Map + bodyParams: Map, + markdownContent: Boolean ) { client.get(ServiceNotifyMessageTemplateResource::class).sendNotifyMessageByTemplate( SendNotifyMessageTemplateRequest( @@ -23,7 +24,8 @@ abstract class NotifySendCmd(val client: Client) : NotifyCmd { titleParams = titleParams, bodyParams = bodyParams, cc = null, - bcc = null + bcc = null, + markdownContent = markdownContent ) ) } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/plugin/svn/service/TriggerSvnService.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/plugin/svn/service/TriggerSvnService.kt index 643a8b1bee8..423f1ebac48 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/plugin/svn/service/TriggerSvnService.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/plugin/svn/service/TriggerSvnService.kt @@ -347,7 +347,8 @@ class TriggerSvnService( svnRevisionInfo.revision.toInt(), paths, files, - svnRevisionInfo.commitTime + svnRevisionInfo.commitTime, + files.size ) } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/plugin/trigger/element/TimerTriggerElementBizPlugin.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/plugin/trigger/element/TimerTriggerElementBizPlugin.kt index a73c708ac4b..7f88d6f1fc6 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/plugin/trigger/element/TimerTriggerElementBizPlugin.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/plugin/trigger/element/TimerTriggerElementBizPlugin.kt @@ -27,7 +27,10 @@ package com.tencent.devops.process.plugin.trigger.element +import com.tencent.bkrepo.common.api.exception.NotFoundException +import com.tencent.devops.common.api.enums.RepositoryType import com.tencent.devops.common.api.enums.ScmType +import com.tencent.devops.common.api.enums.TriggerRepositoryType import com.tencent.devops.common.api.exception.ErrorCodeException import com.tencent.devops.common.client.Client import com.tencent.devops.common.pipeline.container.Container @@ -44,6 +47,7 @@ import com.tencent.devops.process.plugin.ElementBizPlugin import com.tencent.devops.process.plugin.annotation.ElementBiz import com.tencent.devops.process.plugin.trigger.service.PipelineTimerService import com.tencent.devops.process.plugin.trigger.util.CronExpressionUtils +import com.tencent.devops.process.pojo.pipeline.PipelineYamlVo import com.tencent.devops.process.utils.PIPELINE_TIMER_DISABLE import com.tencent.devops.repository.api.ServiceRepositoryResource import com.tencent.devops.repository.pojo.Repository @@ -79,7 +83,8 @@ class TimerTriggerElementBizPlugin constructor( userId: String, channelCode: ChannelCode, create: Boolean, - container: Container + container: Container, + yamlInfo: PipelineYamlVo? ) { val crontabExpressions = mutableSetOf() val params = (container as TriggerContainer).params.associate { it.id to it.defaultValue.toString() } @@ -113,7 +118,7 @@ class TimerTriggerElementBizPlugin constructor( crontabExpressions.add(cron) } } - val repo = getRepo(projectId = projectId, element = element) + val repo = getRepo(projectId = projectId, element = element, params = params, yamlInfo = yamlInfo) // svn仓库分支必填 if (repo != null && repo.getScmType() == ScmType.CODE_SVN && element.branches.isNullOrEmpty()) { throw ErrorCodeException( @@ -150,22 +155,52 @@ class TimerTriggerElementBizPlugin constructor( } } - private fun getRepo(projectId: String, element: TimerTriggerElement): Repository? { + private fun getRepo( + projectId: String, + element: TimerTriggerElement, + params: Map, + yamlInfo: PipelineYamlVo? + ): Repository? { return when { + element.repositoryType == TriggerRepositoryType.SELF -> { + if (yamlInfo == null || yamlInfo.repoHashId.isBlank()) { + throw ErrorCodeException( + errorCode = ProcessMessageCode.ERROR_TIMER_TRIGGER_NEED_ENABLE_PAC + ) + } + try { + client.get(ServiceRepositoryResource::class).get( + projectId = projectId, + repositoryId = yamlInfo.repoHashId, + repositoryType = RepositoryType.ID + ).data + } catch (ignored: NotFoundException) { + throw ErrorCodeException( + errorCode = ProcessMessageCode.ERROR_TIMER_TRIGGER_REPO_NOT_FOUND + ) + } + } + !element.repoHashId.isNullOrBlank() || !element.repoName.isNullOrBlank() -> { val repositoryConfig = with(element) { RepositoryConfigUtils.getRepositoryConfig( repoHashId = repoHashId, repoName = repoName, - repoType = repositoryType, - variables = variables + repoType = TriggerRepositoryType.toRepositoryType(repositoryType), + variables = params + ) + } + try { + client.get(ServiceRepositoryResource::class).get( + projectId = projectId, + repositoryId = repositoryConfig.getURLEncodeRepositoryId(), + repositoryType = repositoryConfig.repositoryType + ).data + } catch (ignored: NotFoundException) { + throw ErrorCodeException( + errorCode = ProcessMessageCode.ERROR_TIMER_TRIGGER_REPO_NOT_FOUND ) } - client.get(ServiceRepositoryResource::class).get( - projectId = projectId, - repositoryId = repositoryConfig.getURLEncodeRepositoryId(), - repositoryType = repositoryConfig.repositoryType - ).data } else -> null } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/plugin/trigger/service/PipelineTimerService.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/plugin/trigger/service/PipelineTimerService.kt index 6218015d485..03bcc27be1e 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/plugin/trigger/service/PipelineTimerService.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/plugin/trigger/service/PipelineTimerService.kt @@ -153,27 +153,29 @@ open class PipelineTimerService @Autowired constructor( } private fun convert(timerRecord: TPipelineTimerRecord): PipelineTimer? { - return PipelineTimer( - projectId = timerRecord.projectId, - pipelineId = timerRecord.pipelineId, - startUser = timerRecord.creator, - crontabExpressions = try { - JsonUtil.to(timerRecord.crontab, object : TypeReference>() {}) - } catch (ignored: Throwable) { - listOf(timerRecord.crontab) - }, - channelCode = try { - ChannelCode.valueOf(timerRecord.channel) - } catch (e: IllegalArgumentException) { - logger.warn("Unkown channel code", e) - return null - }, - repoHashId = timerRecord.repoHashId, - branchs = timerRecord.branchs?.let { - JsonUtil.to(it, object : TypeReference>() {}) - }, - noScm = timerRecord.noScm - ) + with(timerRecord) { + return PipelineTimer( + projectId = projectId, + pipelineId = pipelineId, + startUser = creator, + crontabExpressions = try { + JsonUtil.to(crontab, object : TypeReference>() {}) + } catch (ignored: Throwable) { + listOf(crontab) + }, + channelCode = try { + ChannelCode.valueOf(channel) + } catch (e: IllegalArgumentException) { + logger.warn("Unkown channel code", e) + return null + }, + repoHashId = repoHashId, + branchs = branchs?.let { + JsonUtil.to(it, object : TypeReference>() {}) + }, + noScm = noScm + ) + } } open fun list(start: Int, limit: Int): Result> { diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/plugin/trigger/timer/listener/PipelineTimerBuildListener.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/plugin/trigger/timer/listener/PipelineTimerBuildListener.kt index 8cb2d9bcc85..f1ebafab79a 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/plugin/trigger/timer/listener/PipelineTimerBuildListener.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/plugin/trigger/timer/listener/PipelineTimerBuildListener.kt @@ -30,17 +30,37 @@ package com.tencent.devops.process.plugin.trigger.timer.listener import com.tencent.devops.common.api.enums.RepositoryConfig import com.tencent.devops.common.api.enums.RepositoryType import com.tencent.devops.common.api.exception.OperationException +import com.tencent.devops.common.api.pojo.I18Variable +import com.tencent.devops.common.api.util.JsonUtil import com.tencent.devops.common.event.dispatcher.pipeline.PipelineEventDispatcher import com.tencent.devops.common.event.listener.pipeline.BaseListener +import com.tencent.devops.common.service.trace.TraceTag +import com.tencent.devops.common.web.utils.I18nUtil +import com.tencent.devops.common.webhook.enums.WebhookI18nConstants.TIMING_START_EVENT_DESC import com.tencent.devops.common.webhook.pojo.code.BK_REPO_WEBHOOK_HASH_ID import com.tencent.devops.common.webhook.pojo.code.PIPELINE_WEBHOOK_BRANCH import com.tencent.devops.process.api.service.ServiceTimerBuildResource +import com.tencent.devops.process.constant.ProcessMessageCode.ERROR_PIPELINE_TIMER_BRANCH_IS_EMPTY +import com.tencent.devops.process.constant.ProcessMessageCode.ERROR_PIPELINE_TIMER_BRANCH_NOT_FOUND +import com.tencent.devops.process.constant.ProcessMessageCode.ERROR_PIPELINE_TIMER_BRANCH_NO_CHANGE +import com.tencent.devops.process.constant.ProcessMessageCode.ERROR_PIPELINE_TIMER_BRANCH_UNKNOWN +import com.tencent.devops.process.engine.pojo.PipelineTimer +import com.tencent.devops.process.engine.service.PipelineRepositoryService import com.tencent.devops.process.plugin.trigger.pojo.event.PipelineTimerBuildEvent import com.tencent.devops.process.plugin.trigger.service.PipelineTimerService +import com.tencent.devops.process.pojo.trigger.PipelineTriggerDetailBuilder +import com.tencent.devops.process.pojo.trigger.PipelineTriggerEventBuilder +import com.tencent.devops.process.pojo.trigger.PipelineTriggerFailedMsg +import com.tencent.devops.process.pojo.trigger.PipelineTriggerReason +import com.tencent.devops.process.pojo.trigger.PipelineTriggerReasonDetail +import com.tencent.devops.process.pojo.trigger.PipelineTriggerStatus +import com.tencent.devops.process.pojo.trigger.PipelineTriggerType import com.tencent.devops.process.service.scm.ScmProxyService -import com.tencent.devops.process.yaml.PipelineYamlService +import com.tencent.devops.process.trigger.PipelineTriggerEventService +import org.slf4j.MDC import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Component +import java.time.LocalDateTime /** * MQ实现的流水线原子任务执行事件 @@ -53,7 +73,8 @@ class PipelineTimerBuildListener @Autowired constructor( private val serviceTimerBuildResource: ServiceTimerBuildResource, private val pipelineTimerService: PipelineTimerService, private val scmProxyService: ScmProxyService, - private val pipelineYamlService: PipelineYamlService + private val triggerEventService: PipelineTriggerEventService, + private val pipelineRepositoryService: PipelineRepositoryService ) : BaseListener(pipelineEventDispatcher) { override fun run(event: PipelineTimerBuildEvent) { @@ -61,14 +82,13 @@ class PipelineTimerBuildListener @Autowired constructor( pipelineTimerService.get(projectId = event.projectId, pipelineId = event.pipelineId) ?: return with(pipelineTimer) { when { - (repoHashId == null && branchs.isNullOrEmpty()) || noScm != true -> + repoHashId.isNullOrBlank() -> timerTrigger(event = event) else -> repoTimerTrigger( event = event, - repoHashId = repoHashId, - branchs = branchs + pipelineTimer = pipelineTimer ) } } @@ -102,29 +122,18 @@ class PipelineTimerBuildListener @Autowired constructor( } } - private fun repoTimerTrigger(event: PipelineTimerBuildEvent, repoHashId: String?, branchs: List?) { - with(event) { + @Suppress("NestedBlockDepth") + private fun repoTimerTrigger(event: PipelineTimerBuildEvent, pipelineTimer: PipelineTimer) { + val messages = mutableSetOf() + val branchMessages = mutableMapOf/*branch*/>() + with(pipelineTimer) { try { - val finalRepoHashId = when { - !repoHashId.isNullOrBlank() -> repoHashId - // 分支不为空,如果流水线开启pac,则为开启pac的代码库 - !branchs.isNullOrEmpty() -> { - val pipelineYamlInfo = pipelineYamlService.getPipelineYamlInfo( - projectId = projectId, pipelineId = pipelineId - ) ?: return - pipelineYamlInfo.repoHashId - } - else -> { - logger.info("timer trigger not found repo hashId|$projectId|$pipelineId") - return - } - } - val repositoryConfig = RepositoryConfig( - repositoryHashId = finalRepoHashId, - repositoryName = null, - repositoryType = RepositoryType.ID - ) val finalBranchs = if (branchs.isNullOrEmpty()) { + val repositoryConfig = RepositoryConfig( + repositoryHashId = repoHashId!!, + repositoryName = null, + repositoryType = RepositoryType.ID + ) val defaultBranch = scmProxyService.getDefaultBranch( projectId = projectId, repositoryConfig = repositoryConfig @@ -133,16 +142,58 @@ class PipelineTimerBuildListener @Autowired constructor( } else { branchs } + if (finalBranchs.isNullOrEmpty()) { + logger.info("time scheduled branch not found|$projectId|$pipelineId") + messages.add(I18nUtil.getCodeLanMessage(ERROR_PIPELINE_TIMER_BRANCH_IS_EMPTY)) + return + } finalBranchs.forEach { branch -> - branchTimerTrigger(event = event, repoHashId = finalRepoHashId, branch = branch) + if (noScm == true) { + branchTimerTrigger( + event = event, + repoHashId = repoHashId!!, + branch = branch, + branchMessages = branchMessages + ) + } else { + timerTrigger( + event = event, + params = mapOf( + BK_REPO_WEBHOOK_HASH_ID to repoHashId!!, + PIPELINE_WEBHOOK_BRANCH to branch + ) + ) + } } } catch (ignored: Exception) { - logger.warn("repo timer trigger fail|$projectId|$pipelineId|$repoHashId|$branchs") + logger.warn("repo scheduled trigger fail|$projectId|$pipelineId|$repoHashId|$branchs") + messages.add(ignored.message ?: "scheduled trigger failed") + } + messages.addAll( + branchMessages.map { (messageCode, branchs) -> + I18nUtil.getCodeLanMessage( + messageCode = messageCode, + params = arrayOf(branchs.joinToString(",")) + ) + } + ) + if (messages.isNotEmpty()) { + saveTriggerEvent( + projectId = projectId, + userId = event.userId, + pipelineId = pipelineId, + reasonDetail = PipelineTriggerFailedMsg(JsonUtil.toJson(messages)) + ) } } } - private fun branchTimerTrigger(event: PipelineTimerBuildEvent, repoHashId: String, branch: String) { + private fun branchTimerTrigger( + event: PipelineTimerBuildEvent, + repoHashId: String, + branch: String, + branchMessages: MutableMap> + ) { val repositoryConfig = RepositoryConfig( repositoryHashId = repoHashId, repositoryName = null, @@ -157,7 +208,12 @@ class PipelineTimerBuildListener @Autowired constructor( repositoryConfig = repositoryConfig, branchName = branch, variables = emptyMap() - ).data?.revision ?: return + ).data?.revision ?: run { + branchMessages.computeIfAbsent(ERROR_PIPELINE_TIMER_BRANCH_NOT_FOUND) { + mutableSetOf() + }.add(branch) + return + } val timerBranch = pipelineTimerService.getTimerBranch( projectId = projectId, pipelineId = pipelineId, @@ -181,11 +237,61 @@ class PipelineTimerBuildListener @Autowired constructor( revision = revision ) } else { - logger.info("branch timer trigger fail,revision not change|$pipelineId|$repoHashId|$branch") + logger.info("branch scheduled trigger fail,revision not change|$pipelineId|$repoHashId|$branch") + branchMessages.computeIfAbsent(ERROR_PIPELINE_TIMER_BRANCH_NO_CHANGE) { + mutableSetOf() + }.add(branch) } - } catch (exception: Exception) { - logger.warn("branch timer trigger fail|$projectId|$pipelineId|$repoHashId|$branch", exception) + } catch (ignored: Exception) { + logger.warn("branch scheduled trigger fail|$projectId|$pipelineId|$repoHashId|$branch", ignored) + branchMessages.computeIfAbsent(ERROR_PIPELINE_TIMER_BRANCH_UNKNOWN) { + mutableSetOf() + }.add(branch) } } } + + private fun saveTriggerEvent( + projectId: String, + userId: String, + pipelineId: String, + reasonDetail: PipelineTriggerReasonDetail + ) { + val pipeline = pipelineRepositoryService.getPipelineInfo(projectId, pipelineId = pipelineId) ?: run { + logger.warn("time trigger pipeline not found|$projectId|$pipelineId") + return + } + val requestId = MDC.get(TraceTag.BIZID) + val eventId = triggerEventService.getEventId() + val triggerEventBuilder = PipelineTriggerEventBuilder() + triggerEventBuilder.requestId(requestId) + triggerEventBuilder.projectId(projectId) + triggerEventBuilder.eventId(eventId) + triggerEventBuilder.triggerUser(userId) + triggerEventBuilder.createTime(LocalDateTime.now()) + triggerEventBuilder.triggerType(PipelineTriggerType.TIME_TRIGGER.name) + triggerEventBuilder.eventSource(userId) + triggerEventBuilder.eventType(PipelineTriggerType.TIME_TRIGGER.name) + triggerEventBuilder.eventDesc( + I18Variable( + code = TIMING_START_EVENT_DESC, + params = listOf(userId) + ).toJsonStr() + ) + + val triggerDetailBuilder = PipelineTriggerDetailBuilder() + triggerDetailBuilder.eventId(eventId) + triggerDetailBuilder.projectId(projectId) + triggerDetailBuilder.pipelineId(pipelineId = pipelineId) + triggerDetailBuilder.pipelineName(pipeline.pipelineName) + triggerDetailBuilder.detailId(triggerEventService.getDetailId()) + triggerDetailBuilder.status(PipelineTriggerStatus.FAILED.name) + triggerDetailBuilder.reason(PipelineTriggerReason.TRIGGER_FAILED.name) + triggerDetailBuilder.reasonDetail(reasonDetail) + + triggerEventService.saveEvent( + triggerEvent = triggerEventBuilder.build(), + triggerDetail = triggerDetailBuilder.build() + ) + } } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineInfoFacadeService.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineInfoFacadeService.kt index cbeea42ac4c..24de5d8f79d 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineInfoFacadeService.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineInfoFacadeService.kt @@ -88,6 +88,7 @@ import com.tencent.devops.process.pojo.PipelineCopy import com.tencent.devops.process.pojo.classify.PipelineViewBulkAdd import com.tencent.devops.process.pojo.pipeline.DeletePipelineResult import com.tencent.devops.process.pojo.pipeline.DeployPipelineResult +import com.tencent.devops.process.pojo.pipeline.PipelineYamlVo import com.tencent.devops.process.pojo.template.TemplateType import com.tencent.devops.process.service.label.PipelineGroupService import com.tencent.devops.process.service.pipeline.PipelineSettingFacadeService @@ -97,6 +98,12 @@ import com.tencent.devops.process.template.service.TemplateService import com.tencent.devops.process.yaml.PipelineYamlFacadeService import com.tencent.devops.process.yaml.transfer.aspect.IPipelineTransferAspect import com.tencent.devops.store.api.template.ServiceTemplateResource +import java.net.URLEncoder +import java.util.LinkedList +import java.util.concurrent.TimeUnit +import javax.ws.rs.core.MediaType +import javax.ws.rs.core.Response +import javax.ws.rs.core.StreamingOutput import org.jooq.DSLContext import org.jooq.impl.DSL import org.slf4j.LoggerFactory @@ -104,12 +111,6 @@ import org.springframework.beans.factory.annotation.Autowired import org.springframework.beans.factory.annotation.Value import org.springframework.dao.DuplicateKeyException import org.springframework.stereotype.Service -import java.net.URLEncoder -import java.util.LinkedList -import java.util.concurrent.TimeUnit -import javax.ws.rs.core.MediaType -import javax.ws.rs.core.Response -import javax.ws.rs.core.StreamingOutput @Suppress("ALL") @Service @@ -179,7 +180,7 @@ class PipelineInfoFacadeService @Autowired constructor( ) ) - val targetVersion = pipelineRepositoryService.getPipelineResourceVersion(projectId, pipelineId, version, true) + val targetVersion = pipelineRepositoryService.getPipelineResourceVersion(projectId, pipelineId, version) ?: throw OperationException( I18nUtil.getCodeLanMessage(ILLEGAL_PIPELINE_MODEL_JSON, language = I18nUtil.getLanguage(userId)) ) @@ -205,7 +206,7 @@ class PipelineInfoFacadeService @Autowired constructor( exportStringToFile(targetVersion.yaml ?: "", "${settingInfo.pipelineName}$suffix") } else { val suffix = PipelineStorageType.MODEL.fileSuffix - exportStringToFile(JsonUtil.toJson(modelAndSetting), "${settingInfo.pipelineName}$suffix") + exportStringToFile(JsonUtil.toSortJson(modelAndSetting), "${settingInfo.pipelineName}$suffix") } } @@ -299,7 +300,8 @@ class PipelineInfoFacadeService @Autowired constructor( useSubscriptionSettings: Boolean? = false, useLabelSettings: Boolean? = false, useConcurrencyGroup: Boolean? = false, - description: String? = null + description: String? = null, + yamlInfo: PipelineYamlVo? = null ): DeployPipelineResult { val watcher = Watcher(id = "createPipeline|$projectId|$userId|$channelCode|$checkPermission|$instanceType|$fixPipelineId") @@ -428,13 +430,15 @@ class PipelineInfoFacadeService @Autowired constructor( channelCode = channelCode, create = true, useSubscriptionSettings = useSubscriptionSettings, + useLabelSettings = useLabelSettings, useConcurrencyGroup = useConcurrencyGroup, versionStatus = versionStatus, branchName = branchName, templateId = templateId, description = description, yaml = yaml, - baseVersion = null + baseVersion = null, + yamlInfo = yamlInfo ) pipelineId = result.pipelineId watcher.stop() @@ -551,7 +555,8 @@ class PipelineInfoFacadeService @Autowired constructor( branchName: String, isDefaultBranch: Boolean, description: String? = null, - aspects: LinkedList? = null + aspects: LinkedList? = null, + yamlInfo: PipelineYamlVo? = null ): DeployPipelineResult { val versionStatus = if (isDefaultBranch) { VersionStatus.RELEASED @@ -589,7 +594,8 @@ class PipelineInfoFacadeService @Autowired constructor( yaml = yamlWithVersion, versionStatus = versionStatus, branchName = branchName, - description = description + description = description, + yamlInfo = yamlInfo ) } @@ -602,7 +608,8 @@ class PipelineInfoFacadeService @Autowired constructor( branchName: String, isDefaultBranch: Boolean, description: String? = null, - aspects: LinkedList? = null + aspects: LinkedList? = null, + yamlInfo: PipelineYamlVo? = null ): DeployPipelineResult { val versionStatus = if (isDefaultBranch) { VersionStatus.RELEASED @@ -646,7 +653,8 @@ class PipelineInfoFacadeService @Autowired constructor( savedSetting = savedSetting, versionStatus = versionStatus, branchName = branchName, - description = description + description = description, + yamlInfo = yamlInfo ) } @@ -663,7 +671,7 @@ class PipelineInfoFacadeService @Autowired constructor( if (releaseBranch == true || branchVersionAction != BranchVersionAction.INACTIVE) { // 如果是发布分支版本则直接更新 pipelineRepositoryService.updatePipelineBranchVersion( - projectId, pipelineId, branchName, branchVersionAction, transactionContext + userId, projectId, pipelineId, branchName, branchVersionAction, transactionContext ) } else { // 如果是删除分支版本则判断是否为最后一个版本 @@ -671,7 +679,7 @@ class PipelineInfoFacadeService @Autowired constructor( projectId, pipelineId, branchName ) pipelineRepositoryService.updatePipelineBranchVersion( - projectId, pipelineId, branchName, branchVersionAction, transactionContext + userId, projectId, pipelineId, branchName, branchVersionAction, transactionContext ) val pipelineInfo = pipelineRepositoryService.getPipelineInfo( projectId = projectId, pipelineId = pipelineId, queryDslContext = transactionContext @@ -710,7 +718,7 @@ class PipelineInfoFacadeService @Autowired constructor( if (setting.pipelineAsCodeSettings?.enable == true && !pipelineAsCodeSettings.enable) { // 关闭PAC开关时，将所有分支版本设为 pipelineRepositoryService.updatePipelineBranchVersion( - projectId, pipelineId, null, BranchVersionAction.INACTIVE + userId, projectId, pipelineId, null, BranchVersionAction.INACTIVE ) } @@ -973,7 +981,8 @@ class PipelineInfoFacadeService @Autowired constructor( versionStatus: VersionStatus? = VersionStatus.RELEASED, branchName: String? = null, description: String? = null, - baseVersion: Int? = null + baseVersion: Int? = null, + yamlInfo: PipelineYamlVo? = null ): DeployPipelineResult { if (checkTemplate && templateService.isTemplatePipeline(projectId, pipelineId)) { throw ErrorCodeException( @@ -1075,7 +1084,8 @@ class PipelineInfoFacadeService @Autowired constructor( branchName = branchName, description = description, yaml = yaml, - baseVersion = baseVersion + baseVersion = baseVersion, + yamlInfo = yamlInfo ) // 审计 ActionAuditContext.current() @@ -1153,6 +1163,9 @@ class PipelineInfoFacadeService @Autowired constructor( checkTemplate: Boolean = true, versionStatus: VersionStatus? = VersionStatus.RELEASED ): DeployPipelineResult { + // fix 用户端可能不传入pipelineId和projectId的问题，或者传错的问题 + setting.pipelineId = pipelineId + setting.projectId = projectId val savedSetting = pipelineSettingFacadeService.saveSetting( userId = userId, projectId = projectId, @@ -1176,7 +1189,6 @@ class PipelineInfoFacadeService @Autowired constructor( if (setting.projectId.isBlank()) { setting.projectId = projectId } - setting.pipelineId = pipelineResult.pipelineId // fix 用户端可能不传入pipelineId的问题，或者传错的问题 return pipelineResult } @@ -1459,6 +1471,34 @@ class PipelineInfoFacadeService @Autowired constructor( return failUpdateModels } + fun locked(userId: String, projectId: String, pipelineId: String, locked: Boolean): PipelineInfo { + val language = I18nUtil.getLanguage(userId) + val permission = AuthPermission.EDIT + pipelinePermissionService.validPipelinePermission( + userId = userId, + projectId = projectId, + pipelineId = pipelineId, + permission = permission, + message = MessageUtil.getMessageByLocale( + messageCode = USER_NOT_PERMISSIONS_OPERATE_PIPELINE, + language = language, + params = arrayOf(userId, projectId, permission.getI18n(language), pipelineId) + ) + ) + + val info = pipelineRepositoryService.getPipelineInfo(projectId, pipelineId) + ?: throw ErrorCodeException( + statusCode = Response.Status.NOT_FOUND.statusCode, + errorCode = ProcessMessageCode.ERROR_PIPELINE_NOT_EXISTS + ) + + if (!pipelineRepositoryService.updateLocked(userId, projectId, pipelineId, locked)) { // 可能重复操作，不打扰用户 + logger.warn("Locked Pipeline|$userId|$projectId|$pipelineId|locked=$locked, may be duplicated") + } + + return info + } + companion object { private val logger = LoggerFactory.getLogger(PipelineInfoFacadeService::class.java) } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineListFacadeService.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineListFacadeService.kt index 38fb154d644..43f68d98e1d 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineListFacadeService.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineListFacadeService.kt @@ -54,7 +54,6 @@ import com.tencent.devops.common.pipeline.enums.PipelineInstanceTypeEnum import com.tencent.devops.common.pipeline.enums.StartType import com.tencent.devops.common.pipeline.enums.VersionStatus import com.tencent.devops.common.pipeline.pojo.element.trigger.enums.CodeEventType -import com.tencent.devops.common.pipeline.pojo.setting.PipelineRunLockType import com.tencent.devops.common.service.utils.LogUtils import com.tencent.devops.common.web.utils.I18nUtil import com.tencent.devops.model.process.tables.TPipelineSetting @@ -468,6 +467,22 @@ class PipelineListFacadeService @Autowired constructor( * 根据视图ID获取流水线编 * 其中 PIPELINE_VIEW_FAVORITE_PIPELINES，PIPELINE_VIEW_MY_PIPELINES，PIPELINE_VIEW_ALL_PIPELINES * 分别对应 我的收藏，我的流水线，全部流水线 + * @param userId 用户ID + * @param projectId 项目ID + * @param page 页码（可选） + * @param pageSize 每页大小（可选） + * @param sortType 流水线排序类型 + * @param channelCode 渠道代码 + * @param viewId 视图ID + * @param checkPermission 是否检查权限，默认为true + * @param filterByPipelineName 根据流水线名称过滤（可选） + * @param filterByCreator 根据创建者过滤（可选） + * @param filterByLabels 根据标签过滤（可选） + * @param filterInvalid 是否过滤无效流水线，默认为false + * @param filterByViewIds 根据视图ID过滤（可选） + * @param collation 流水线排序规则，默认为DEFAULT + * @param showDelete 是否显示已删除的流水线，默认为false + * @param queryByWeb 默认为false, 从页面访问该方法时为true, 会针对页面访问做特殊处理。 */ fun listViewPipelines( userId: String, @@ -484,7 +499,8 @@ class PipelineListFacadeService @Autowired constructor( filterInvalid: Boolean = false, filterByViewIds: String? = null, collation: PipelineCollation = PipelineCollation.DEFAULT, - showDelete: Boolean = false + showDelete: Boolean = false, + queryByWeb: Boolean = false ): PipelineViewPipelinePage { val watcher = Watcher(id = "listViewPipelines|$projectId|$userId") watcher.start("perm_r_perm") @@ -589,7 +605,8 @@ class PipelineListFacadeService @Autowired constructor( pageSize = pageSize, includeDelete = true, collation = collation, - userId = userId + userId = userId, + queryByWeb = queryByWeb ) } else if ((null != page && null != pageSize) && !(page == 1 && pageSize == -1)) { // 判断可用的流水线是否已到最后一页 @@ -611,7 +628,8 @@ class PipelineListFacadeService @Autowired constructor( pageSize = pageSize, includeDelete = includeDelete, collation = collation, - userId = userId + userId = userId, + queryByWeb = queryByWeb ) } else if (page == totalAvailablePipelinePage && totalAvailablePipelineSize > 0) { // 查询可用流水线最后一页不满页的数量 @@ -631,7 +649,8 @@ class PipelineListFacadeService @Autowired constructor( pageSize = pageSize, includeDelete = includeDelete, collation = collation, - userId = userId + userId = userId, + queryByWeb = queryByWeb ) // 可用流水线最后一页不满页的数量需用不可用的流水线填充 if (lastPageRemainNum > 0 && totalInvalidPipelineSize > 0) { @@ -650,7 +669,8 @@ class PipelineListFacadeService @Autowired constructor( pageSize = lastPageRemainNum.toInt(), includeDelete = includeDelete, collation = collation, - userId = userId + userId = userId, + queryByWeb = queryByWeb ) } } else if (totalInvalidPipelineSize > 0) { @@ -673,7 +693,8 @@ class PipelineListFacadeService @Autowired constructor( pageOffsetNum = lastPageRemainNum.toInt(), includeDelete = includeDelete, collation = collation, - userId = userId + userId = userId, + queryByWeb = queryByWeb ) } } else { @@ -693,7 +714,8 @@ class PipelineListFacadeService @Autowired constructor( pageSize = pageSize, includeDelete = includeDelete, collation = collation, - userId = userId + userId = userId, + queryByWeb = queryByWeb ) if (filterInvalid) { @@ -712,7 +734,8 @@ class PipelineListFacadeService @Autowired constructor( pageSize = pageSize, includeDelete = includeDelete, collation = collation, - userId = userId + userId = userId, + queryByWeb = queryByWeb ) } } @@ -904,7 +927,8 @@ class PipelineListFacadeService @Autowired constructor( pageOffsetNum: Int? = 0, includeDelete: Boolean? = false, collation: PipelineCollation = PipelineCollation.DEFAULT, - userId: String + userId: String, + queryByWeb: Boolean = false ) { val pipelineRecords = pipelineBuildSummaryDao.listPipelineInfoBuildSummary( dslContext = dslContext, @@ -929,7 +953,8 @@ class PipelineListFacadeService @Autowired constructor( pipelineInfoRecords = pipelineRecords, favorPipelines = favorPipelines, authPipelines = authPipelines, - projectId = projectId + projectId = projectId, + queryByWeb = queryByWeb ) ) } @@ -1302,7 +1327,8 @@ class PipelineListFacadeService @Autowired constructor( authPipelines: List = emptyList(), excludePipelineId: String? = null, projectId: String, - queryModelFlag: Boolean? = false + queryModelFlag: Boolean? = false, + queryByWeb: Boolean = false ): MutableList { // 初始化信息 val pipelines = mutableListOf() @@ -1402,7 +1428,8 @@ class PipelineListFacadeService @Autowired constructor( pipelineBuildMap = pipelineBuildMap, buildTaskTotalCountMap = buildTaskTotalCountMap, buildTaskFinishCountMap = buildTaskFinishCountMap, - pipelineYamlExistMap = pipelineYamlExistMap + pipelineYamlExistMap = pipelineYamlExistMap, + queryByWeb = queryByWeb ) return pipelines @@ -1419,7 +1446,8 @@ class PipelineListFacadeService @Autowired constructor( pipelineBuildMap: Map, buildTaskTotalCountMap: Map, buildTaskFinishCountMap: Map, - pipelineYamlExistMap: Map + pipelineYamlExistMap: Map, + queryByWeb: Boolean ) { pipelines.forEach { val pipelineId = it.pipelineId @@ -1448,6 +1476,13 @@ class PipelineListFacadeService @Autowired constructor( it.viewNames = pipelineViewNameMap[it.pipelineId] } pipelineBuildMap[pipelineId]?.let { lastBuild -> + if (queryByWeb) { + /*针对web页面的访问特殊覆盖这部分数据*/ + it.latestBuildNum = lastBuild.buildNum + it.latestBuildNumAlias = lastBuild.buildNumAlias + it.latestBuildStartTime = lastBuild.startTime + it.latestBuildEndTime = lastBuild.endTime + } it.lastBuildMsg = BuildMsgUtils.getBuildMsg( buildMsg = lastBuild.buildMsg, startType = StartType.toStartType(lastBuild.trigger), @@ -1491,7 +1526,6 @@ class PipelineListFacadeService @Autowired constructor( if (pipelineSettingRecord != null) { val tSetting = TPipelineSetting.T_PIPELINE_SETTING it.pipelineDesc = pipelineSettingRecord.get(tSetting.DESC) - it.lock = PipelineRunLockType.checkLock(pipelineSettingRecord.get(tSetting.RUN_LOCK_TYPE)) it.buildNumRule = pipelineSettingRecord.get(tSetting.BUILD_NUM_RULE) } it.yamlExist = pipelineYamlExistMap[pipelineId] ?: false @@ -1531,6 +1565,7 @@ class PipelineListFacadeService @Autowired constructor( pipelineId = pipelineId, pipelineName = it.pipelineName, taskCount = it.taskCount, + lock = it.locked, canManualStartup = it.manualStartup == 1, latestBuildEstimatedExecutionSeconds = latestBuildEstimatedExecutionSeconds, deploymentTime = (it.updateTime)?.timestampmilli() ?: 0, @@ -1873,7 +1908,8 @@ class PipelineListFacadeService @Autowired constructor( createTime = pipelineInfo.createTime, updateTime = pipelineInfo.updateTime, viewNames = pipelineViewNames, - latestVersionStatus = pipelineInfo.latestVersionStatus + latestVersionStatus = pipelineInfo.latestVersionStatus, + locked = pipelineInfo.locked ?: false ) } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineVersionFacadeService.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineVersionFacadeService.kt index 8ffddae94b7..4a00709c108 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineVersionFacadeService.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/PipelineVersionFacadeService.kt @@ -34,6 +34,7 @@ import com.tencent.devops.common.api.pojo.Page import com.tencent.devops.common.api.pojo.PipelineAsCodeSettings import com.tencent.devops.common.api.util.PageUtil import com.tencent.devops.common.api.util.timestampmilli +import com.tencent.devops.common.log.utils.BuildLogPrinter import com.tencent.devops.common.pipeline.Model import com.tencent.devops.common.pipeline.PipelineVersionWithModel import com.tencent.devops.common.pipeline.PipelineVersionWithModelRequest @@ -53,8 +54,10 @@ import com.tencent.devops.common.pipeline.pojo.transfer.PreviewResponse import com.tencent.devops.common.pipeline.pojo.transfer.TransferActionType import com.tencent.devops.common.pipeline.pojo.transfer.TransferBody import com.tencent.devops.common.pipeline.pojo.transfer.YamlWithVersion +import com.tencent.devops.common.redis.RedisOperation import com.tencent.devops.common.web.utils.I18nUtil import com.tencent.devops.process.constant.ProcessMessageCode +import com.tencent.devops.process.engine.control.lock.PipelineReleaseLock import com.tencent.devops.process.engine.dao.PipelineBuildDao import com.tencent.devops.process.engine.dao.PipelineBuildSummaryDao import com.tencent.devops.process.engine.pojo.PipelineVersionWithInfo @@ -75,16 +78,17 @@ import com.tencent.devops.process.template.service.TemplateService import com.tencent.devops.process.utils.PipelineVersionUtils import com.tencent.devops.process.yaml.PipelineYamlFacadeService import com.tencent.devops.process.yaml.transfer.PipelineTransferException -import javax.ws.rs.core.Response import org.jooq.DSLContext import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service +import javax.ws.rs.core.Response @Suppress("ALL") @Service class PipelineVersionFacadeService @Autowired constructor( private val dslContext: DSLContext, + private val redisOperation: RedisOperation, private val modelCheckPlugin: ModelCheckPlugin, private val pipelineListFacadeService: PipelineListFacadeService, private val pipelineSettingFacadeService: PipelineSettingFacadeService, @@ -100,7 +104,8 @@ class PipelineVersionFacadeService @Autowired constructor( private val pipelineGroupService: PipelineGroupService, private val pipelineViewGroupService: PipelineViewGroupService, private val pipelineBuildSummaryDao: PipelineBuildSummaryDao, - private val pipelineBuildDao: PipelineBuildDao + private val pipelineBuildDao: PipelineBuildDao, + private val buildLogPrinter: BuildLogPrinter ) { companion object { @@ -157,8 +162,28 @@ class PipelineVersionFacadeService @Autowired constructor( pipelineId = pipelineId, detailInfo = detailInfo ) - val version = draftVersion?.version ?: releaseVersion.version - val versionName = draftVersion?.versionName ?: releaseVersion.versionName + val released = detailInfo.latestVersionStatus?.isNotReleased() != true + var versionName = releaseVersion.versionName?.takeIf { released } + // 配合前端的展示需要，version有以下几种情况的返回值： + // 1 发布过且有草稿：version取草稿的版本号 + // 2 发布过且有分支版本：version取最新正式的版本号 + // 3 未发布过仅有草稿版本：version取草稿的版本号 + // 4 未发布过仅有分支版本：version取最新的分支版本号 + val version = when (detailInfo.latestVersionStatus) { + VersionStatus.COMMITTING -> { + draftVersion?.version + } + VersionStatus.BRANCH -> { + val branchVersion = pipelineRepositoryService.getBranchVersionResource( + projectId, pipelineId, null + ) + versionName = branchVersion?.versionName + branchVersion?.version + } + else -> { + draftVersion?.version + } + } ?: releaseVersion.version val permissions = pipelineListFacadeService.getPipelinePermissions(userId, projectId, pipelineId) val yamlExist = pipelineYamlFacadeService.yamlExistInDefaultBranch( projectId = projectId, @@ -171,6 +196,7 @@ class PipelineVersionFacadeService @Autowired constructor( hasCollect = detailInfo.hasCollect, instanceFromTemplate = detailInfo.instanceFromTemplate, templateId = detailInfo.templateId, + templateVersion = detailInfo.templateVersion, canManualStartup = detailInfo.canManualStartup, canDebug = canDebug, canRelease = canRelease, @@ -185,14 +211,16 @@ class PipelineVersionFacadeService @Autowired constructor( permissions = permissions, version = version, versionName = versionName, - releaseVersion = releaseVersion.version, - releaseVersionName = releaseVersion.versionName, + // 前端需要缺省当前能用的版本，用于进入页面的默认展示，但没有发布过就不提供releaseVersionName + releaseVersion = releaseVersion.version.takeIf { released } ?: version, + releaseVersionName = releaseVersion.versionName?.takeIf { released }, baseVersion = baseVersion, baseVersionStatus = baseVersionStatus, baseVersionName = baseVersionName, pipelineAsCodeSettings = PipelineAsCodeSettings(enable = yamlInfo != null), yamlInfo = yamlInfo, - yamlExist = yamlExist + yamlExist = yamlExist, + locked = detailInfo.locked ) } @@ -252,210 +280,244 @@ class PipelineVersionFacadeService @Autowired constructor( version: Int, request: PipelineVersionReleaseRequest ): DeployPipelineResult { - val pipeline = pipelineRepositoryService.getPipelineInfo(projectId, pipelineId) - ?: throw ErrorCodeException( - statusCode = Response.Status.NOT_FOUND.statusCode, - errorCode = ProcessMessageCode.ERROR_PIPELINE_NOT_EXISTS - ) + PipelineReleaseLock(redisOperation, pipelineId).use { + val pipeline = pipelineRepositoryService.getPipelineInfo(projectId, pipelineId) + ?: throw ErrorCodeException( + statusCode = Response.Status.NOT_FOUND.statusCode, + errorCode = ProcessMessageCode.ERROR_PIPELINE_NOT_EXISTS + ) // if (templateService.isTemplatePipeline(projectId, pipelineId)) { // throw ErrorCodeException( // errorCode = ProcessMessageCode.ERROR_PIPELINE_TEMPLATE_CAN_NOT_EDIT // ) // } - val draftVersion = pipelineRepositoryService.getPipelineResourceVersion( - projectId = projectId, - pipelineId = pipelineId, - version = version, - includeDraft = true - ) ?: throw ErrorCodeException( - errorCode = ProcessMessageCode.ERROR_NO_PIPELINE_DRAFT_EXISTS - ) - val originSetting = draftVersion.settingVersion?.let { - pipelineSettingFacadeService.userGetSetting( + val draftVersion = pipelineRepositoryService.getPipelineResourceVersion( + projectId = projectId, + pipelineId = pipelineId, + version = version, + includeDraft = true + ) ?: throw ErrorCodeException( + errorCode = ProcessMessageCode.ERROR_NO_PIPELINE_DRAFT_EXISTS + ) + val originSetting = draftVersion.settingVersion?.let { + pipelineSettingFacadeService.userGetSetting( + userId = userId, + projectId = projectId, + pipelineId = pipelineId, + version = it + ) + } ?: pipelineSettingFacadeService.userGetSetting( userId = userId, projectId = projectId, + pipelineId = pipelineId + ) + // 提前初始化检查一次让编排报错，避免PAC代码库操作后报错 + pipelineRepositoryService.initModel( + model = draftVersion.model, projectId = projectId, pipelineId = pipelineId, - version = it + userId = userId, + create = false, + versionStatus = VersionStatus.RELEASED, + channelCode = pipeline.channelCode, + yamlInfo = request.yamlInfo ) - } ?: pipelineSettingFacadeService.userGetSetting( - userId = userId, - projectId = projectId, - pipelineId = pipelineId - ) - // 提前初始化检查一次让编排报错，避免PAC代码库操作后报错 - pipelineRepositoryService.initModel( - model = draftVersion.model, projectId = projectId, - pipelineId = pipelineId, - userId = userId, - create = false, - versionStatus = VersionStatus.RELEASED, - channelCode = pipeline.channelCode - ) - val originYaml = pipelineYamlFacadeService.getPipelineYamlInfo(projectId, pipelineId, version) - // 如果不匹配已有状态则报错，需要用户重新刷新页面 - if (originYaml != null && !request.enablePac) throw ErrorCodeException( - errorCode = ProcessMessageCode.ERROR_PIPELINE_IS_NOT_THE_LATEST - ) - // 根据项目PAC状态进行接口调用 - val enabled = originYaml != null || request.enablePac - val targetSettings = originSetting.copy( - pipelineAsCodeSettings = PipelineAsCodeSettings(enabled) - ) - var targetUrl: String? = null - val (versionStatus, branchName) = if ( - enabled && request.targetAction == CodeTargetAction.CHECKOUT_BRANCH_AND_REQUEST_MERGE - ) { - Pair(VersionStatus.BRANCH_RELEASE, getReleaseBranchName(pipelineId, draftVersion.version)) - } else if (enabled && request.targetAction == CodeTargetAction.PUSH_BRANCH_AND_REQUEST_MERGE) { - val baseVersion = draftVersion.baseVersion?.let { - pipelineRepositoryService.getPipelineResourceVersion(projectId, pipelineId, it) - } - if (baseVersion == null || baseVersion.status != VersionStatus.BRANCH) throw ErrorCodeException( - errorCode = ProcessMessageCode.ERROR_NO_PIPELINE_VERSION_EXISTS_BY_ID, - params = arrayOf(draftVersion.baseVersion?.toString() ?: "") + val originYaml = pipelineYamlFacadeService.getPipelineYamlInfo(projectId, pipelineId, version) + // 如果不匹配已有状态则报错，需要用户重新刷新页面 + if (originYaml != null && !request.enablePac) throw ErrorCodeException( + errorCode = ProcessMessageCode.ERROR_PIPELINE_IS_NOT_THE_LATEST ) - Pair(VersionStatus.BRANCH_RELEASE, baseVersion.versionName) - } else { - Pair(VersionStatus.DRAFT_RELEASE, null) - } - if (enabled) { - if (request.yamlInfo == null) throw ErrorCodeException( - errorCode = CommonMessageCode.ERROR_NEED_PARAM_, - params = arrayOf(PipelineVersionReleaseRequest::yamlInfo.name) + // 根据项目PAC状态进行接口调用 + val enabled = originYaml != null || request.enablePac + val targetSettings = originSetting.copy( + pipelineAsCodeSettings = PipelineAsCodeSettings(enabled) ) - if (draftVersion.yaml.isNullOrBlank()) { - transferService.transfer( + val (versionStatus, branchName) = if ( + enabled && request.targetAction == CodeTargetAction.CHECKOUT_BRANCH_AND_REQUEST_MERGE + ) { + Pair(VersionStatus.BRANCH_RELEASE, getReleaseBranchName(pipelineId, draftVersion.version)) + } else if (enabled && request.targetAction == CodeTargetAction.PUSH_BRANCH_AND_REQUEST_MERGE) { + val baseVersion = draftVersion.baseVersion?.let { + pipelineRepositoryService.getPipelineResourceVersion(projectId, pipelineId, it) + } + if (baseVersion == null || baseVersion.status != VersionStatus.BRANCH) throw ErrorCodeException( + errorCode = ProcessMessageCode.ERROR_NO_PIPELINE_VERSION_EXISTS_BY_ID, + params = arrayOf(draftVersion.baseVersion?.toString() ?: "") + ) + Pair(VersionStatus.BRANCH_RELEASE, baseVersion.versionName) + } else { + Pair(VersionStatus.DRAFT_RELEASE, null) + } + val targetAction = request.targetAction ?: CodeTargetAction.PUSH_BRANCH_AND_REQUEST_MERGE + if (enabled) { + if (request.yamlInfo == null) throw ErrorCodeException( + errorCode = CommonMessageCode.ERROR_NEED_PARAM_, + params = arrayOf(PipelineVersionReleaseRequest::yamlInfo.name) + ) + if (draftVersion.yaml.isNullOrBlank()) { + transferService.transfer( + userId = userId, + projectId = projectId, + pipelineId = pipelineId, + actionType = TransferActionType.FULL_MODEL2YAML, + data = TransferBody( + modelAndSetting = PipelineModelAndSetting(draftVersion.model, targetSettings), + yamlFileName = request.yamlInfo?.filePath + ) + ) + throw ErrorCodeException( + errorCode = ProcessMessageCode.ERROR_YAML_CONTENT_IS_EMPTY + ) + } + val yamlInfo = request.yamlInfo ?: throw ErrorCodeException( + errorCode = CommonMessageCode.ERROR_NEED_PARAM_, + params = arrayOf(PipelineVersionReleaseRequest::yamlInfo.name) + ) + // 对前端的YAML信息进行校验 + val filePath = if (yamlInfo.filePath.endsWith(".yaml") || yamlInfo.filePath.endsWith(".yml")) { + yamlInfo.filePath + } else { + throw ErrorCodeException( + errorCode = ProcessMessageCode.ERROR_PIPELINE_YAML_FILENAME, + params = arrayOf(yamlInfo.filePath) + ) + } + pipelineYamlFacadeService.checkPushParam( userId = userId, projectId = projectId, pipelineId = pipelineId, - actionType = TransferActionType.FULL_MODEL2YAML, - data = TransferBody( - modelAndSetting = PipelineModelAndSetting(draftVersion.model, targetSettings), - yamlFileName = request.yamlInfo?.filePath - ) - ) - throw ErrorCodeException( - errorCode = ProcessMessageCode.ERROR_YAML_CONTENT_IS_EMPTY + version = draftVersion.version, + versionName = branchName, + repoHashId = yamlInfo.repoHashId, + scmType = yamlInfo.scmType!!, + filePath = filePath, + content = draftVersion.yaml ?: "", + targetAction = targetAction ) } - val yamlInfo = request.yamlInfo ?: throw ErrorCodeException( - errorCode = CommonMessageCode.ERROR_NEED_PARAM_, - params = arrayOf(PipelineVersionReleaseRequest::yamlInfo.name) + + val savedSetting = pipelineSettingFacadeService.saveSetting( + userId = userId, + projectId = projectId, + pipelineId = pipelineId, + updateVersion = false, + versionStatus = versionStatus, + setting = targetSettings ) - // 对前端的YAML信息进行校验 - val filePath = if (yamlInfo.filePath.endsWith(".yaml") || yamlInfo.filePath.endsWith(".yml")) { - yamlInfo.filePath - } else { - throw ErrorCodeException( - errorCode = ProcessMessageCode.ERROR_PIPELINE_YAML_FILENAME, - params = arrayOf(yamlInfo.filePath) + if (versionStatus.isReleasing()) { + val existModel = pipelineRepositoryService.getPipelineResourceVersion( + projectId = projectId, + pipelineId = pipelineId + )?.model ?: throw ErrorCodeException( + statusCode = Response.Status.NOT_FOUND.statusCode, + errorCode = ProcessMessageCode.ERROR_PIPELINE_MODEL_NOT_EXISTS + ) + // 对已经存在的模型做处理 + val param = BeforeDeleteParam( + userId = userId, + projectId = projectId, + pipelineId = pipelineId ) + modelCheckPlugin.beforeDeleteElementInExistsModel(existModel, draftVersion.model, param) } - val targetAction = request.targetAction ?: CodeTargetAction.PUSH_BRANCH_AND_REQUEST_MERGE - // #8161 如果调用代码库同步失败则有报错或提示 - val pushResult = pipelineYamlFacadeService.pushYamlFile( + val result = pipelineRepositoryService.deployPipeline( + model = draftVersion.model, + projectId = projectId, + signPipelineId = pipelineId, + userId = userId, + channelCode = pipeline.channelCode, + create = false, + updateLastModifyUser = true, + setting = savedSetting, + versionStatus = versionStatus, + branchName = branchName, + description = request.description?.takeIf { it.isNotBlank() } ?: draftVersion.description, + yaml = YamlWithVersion(versionTag = draftVersion.yamlVersion, yamlStr = draftVersion.yaml), + baseVersion = draftVersion.baseVersion, + yamlInfo = request.yamlInfo + ) + // 添加标签 + pipelineGroupService.addPipelineLabel( userId = userId, projectId = projectId, pipelineId = pipelineId, - version = draftVersion.version, - versionName = branchName, - pipelineName = targetSettings.pipelineName, - content = draftVersion.yaml ?: "", - commitMessage = request.description ?: "update", - repoHashId = yamlInfo.repoHashId, - scmType = yamlInfo.scmType, - filePath = filePath, - targetAction = targetAction + labelIds = targetSettings.labels ) - targetUrl = pushResult.mrUrl - } - val savedSetting = pipelineSettingFacadeService.saveSetting( - userId = userId, - projectId = projectId, - pipelineId = pipelineId, - updateVersion = false, - versionStatus = versionStatus, - setting = targetSettings - ) - if (versionStatus.isReleasing()) { - val existModel = pipelineRepositoryService.getPipelineResourceVersion( + // 添加到动态分组 + pipelineViewGroupService.updateGroupAfterPipelineUpdate( projectId = projectId, pipelineId = pipelineId, - includeDraft = true - )?.model ?: throw ErrorCodeException( - statusCode = Response.Status.NOT_FOUND.statusCode, - errorCode = ProcessMessageCode.ERROR_PIPELINE_MODEL_NOT_EXISTS + userId = userId, + creator = userId, + pipelineName = savedSetting.pipelineName ) - // 对已经存在的模型做处理 - val param = BeforeDeleteParam( + // 添加到静态流水线组 + pipelineViewGroupService.bulkAddStatic( userId = userId, projectId = projectId, - pipelineId = pipelineId + pipelineId = pipelineId, + staticViewIds = request.staticViews ) - modelCheckPlugin.beforeDeleteElementInExistsModel(existModel, draftVersion.model, param) - } - val result = pipelineRepositoryService.deployPipeline( - model = draftVersion.model, - projectId = projectId, - signPipelineId = pipelineId, - userId = draftVersion.creator, - channelCode = pipeline.channelCode, - create = false, - updateLastModifyUser = true, - setting = savedSetting, - versionStatus = versionStatus, - branchName = branchName, - description = request.description?.takeIf { it.isNotBlank() } ?: draftVersion.description, - yaml = YamlWithVersion(versionTag = draftVersion.yamlVersion, yamlStr = draftVersion.yaml), - baseVersion = draftVersion.baseVersion - ) - // 添加标签 - pipelineGroupService.addPipelineLabel( - userId = userId, - projectId = projectId, - pipelineId = pipelineId, - labelIds = targetSettings.labels - ) - // 添加到动态分组 - pipelineViewGroupService.updateGroupAfterPipelineUpdate( - projectId = projectId, - pipelineId = pipelineId, - userId = userId, - creator = userId, - pipelineName = savedSetting.pipelineName - ) - // 添加到静态流水线组 - pipelineViewGroupService.bulkAddStatic( - userId = userId, - projectId = projectId, - pipelineId = pipelineId, - staticViewIds = request.staticViews - ) - // #8164 发布后的流水将调试信息清空为0，重新计数，同时取消该版本的调试记录 - pipelineBuildDao.getDebugHistory(dslContext, projectId, pipelineId, version).forEach { - if (!it.status.isFinish()) pipelineBuildFacadeService.serviceShutdown( - projectId, pipelineId, it.buildId, pipeline.channelCode + // #8164 发布后的流水将调试信息清空为0，重新计数，同时取消该版本的调试记录 + pipelineBuildDao.getDebugHistory(dslContext, projectId, pipelineId).forEach { debug -> + if (!debug.status.isFinish()) { + buildLogPrinter.addWarnLine( + buildId = debug.buildId, executeCount = debug.executeCount ?: 1, + tag = "", jobId = null, stepId = null, + message = "" + ) + pipelineBuildFacadeService.buildManualShutdown( + userId = userId, projectId = projectId, pipelineId = pipelineId, + buildId = debug.buildId, channelCode = pipeline.channelCode, terminateFlag = true + ) + } + } + pipelineBuildSummaryDao.resetDebugInfo(dslContext, projectId, pipelineId) + pipelineBuildDao.clearDebugHistory(dslContext, projectId, pipelineId) + + var targetUrl: String? = null + // 推送代码库应该在流水线变更成功之后,蓝盾发布形成闭环,如果代码库推送失败,应该引导用户手工修复 + if (enabled) { + val yamlInfo = request.yamlInfo!! + // #8161 如果调用代码库同步失败则有报错或提示 + val pushResult = pipelineYamlFacadeService.pushYamlFile( + userId = userId, + projectId = projectId, + pipelineId = pipelineId, + version = draftVersion.version, + versionName = branchName, + pipelineName = targetSettings.pipelineName, + content = draftVersion.yaml ?: "", + commitMessage = request.description ?: "update", + repoHashId = yamlInfo.repoHashId, + scmType = yamlInfo.scmType!!, + filePath = yamlInfo.filePath, + targetAction = targetAction + ) + targetUrl = pushResult.mrUrl + } + val yamlInfo = pipelineYamlFacadeService.getPipelineYamlInfo(projectId, pipelineId, version) + return DeployPipelineResult( + pipelineId = pipelineId, + pipelineName = draftVersion.model.name, + version = result.version, + versionNum = result.versionNum, + versionName = result.versionName, + targetUrl = targetUrl, + yamlInfo = yamlInfo ) } - pipelineBuildSummaryDao.resetDebugInfo(dslContext, projectId, pipelineId) - pipelineBuildDao.clearDebugHistory(dslContext, projectId, pipelineId, version) - return DeployPipelineResult( - pipelineId = pipelineId, - pipelineName = draftVersion.model.name, - version = result.version, - versionNum = result.versionNum, - versionName = result.versionName, - targetUrl = targetUrl - ) } - fun createPipelineFromTemplate( + /** + * 从自由模式下创建流水线 + */ + fun createPipelineFromFreedom( userId: String, projectId: String, request: TemplateInstanceCreateRequest ): DeployPipelineResult { - val (templateModel, instanceFromTemplate) = if (request.emptyTemplate == true) { - val model = Model( + val templateModel = if (request.emptyTemplate == true) { + Model( name = request.pipelineName, desc = "", stages = listOf( @@ -481,15 +543,13 @@ class PipelineVersionFacadeService @Autowired constructor( ), pipelineCreator = userId ) - Pair(model, true) } else { - val template = templateFacadeService.getTemplate( + templateFacadeService.getTemplate( userId = userId, projectId = projectId, templateId = request.templateId, version = request.templateVersion - ) - Pair(template.template, true) + ).template } return pipelineInfoFacadeService.createPipeline( userId = userId, @@ -497,8 +557,7 @@ class PipelineVersionFacadeService @Autowired constructor( model = templateModel.copy( name = request.pipelineName, templateId = request.templateId, - instanceFromTemplate = instanceFromTemplate, - labels = request.labels, + instanceFromTemplate = false, staticViews = request.staticViews ), channelCode = ChannelCode.BS, @@ -535,7 +594,7 @@ class PipelineVersionFacadeService @Autowired constructor( userId = userId, projectId = projectId, pipelineId = pipelineId, - version = resource.settingVersion ?: version + version = resource.settingVersion ?: 0 // 历史没有关联过setting版本应该取正式版本 ) val model = pipelineInfoFacadeService.getFixedModel( model = resource.model, @@ -731,7 +790,7 @@ class PipelineVersionFacadeService @Autowired constructor( includeDraft = includeDraft ) } else null - val (size, pipelines) = repositoryVersionService.listPipelineVersion( + val (size, pipelines) = repositoryVersionService.listPipelineVersionWithInfo( pipelineInfo = pipelineInfo, projectId = projectId, pipelineId = pipelineId, @@ -770,6 +829,69 @@ class PipelineVersionFacadeService @Autowired constructor( ) } + fun listPipelineVersion( + projectId: String, + pipelineId: String, + page: Int, + pageSize: Int, + fromVersion: Int?, + includeDraft: Boolean? = true, + versionName: String? = null, + creator: String? = null, + description: String? = null + ): Page { + var slqLimit: SQLLimit? = null + if (pageSize != -1) slqLimit = PageUtil.convertPageSizeToSQLLimit(page, pageSize) + + val offset = slqLimit?.offset ?: 0 + var limit = slqLimit?.limit ?: -1 + // 如果有要插队的版本需要提到第一页，则在查询list时排除，单独查出来放在第一页 + val fromResource = if (fromVersion != null && page == 1) { + limit -= 1 + repositoryVersionService.getPipelineVersionSimple( + projectId = projectId, + pipelineId = pipelineId, + version = fromVersion + ) + } else null + val pipelineInfo = pipelineRepositoryService.getPipelineInfo(projectId, pipelineId) + val (size, pipelines) = repositoryVersionService.listPipelineVersion( + pipelineInfo = pipelineInfo, + projectId = projectId, + pipelineId = pipelineId, + creator = creator, + description = description, + versionName = versionName, + includeDraft = includeDraft, + excludeVersion = fromVersion, + offset = offset, + limit = limit + ) + fromResource?.let { pipelines.add(it) } + return Page( + page = page, + pageSize = pageSize, + count = size.toLong(), + records = pipelines + ) + } + + fun getPipelineVersion( + projectId: String, + pipelineId: String, + version: Int + ): PipelineVersionSimple { + return repositoryVersionService.getPipelineVersionSimple( + projectId = projectId, + pipelineId = pipelineId, + version = version + ) ?: throw ErrorCodeException( + statusCode = Response.Status.NOT_FOUND.statusCode, + errorCode = ProcessMessageCode.ERROR_NO_PIPELINE_VERSION_EXISTS_BY_ID, + params = arrayOf(version.toString()) + ) + } + fun rollbackDraftFromVersion( userId: String, projectId: String, @@ -786,6 +908,7 @@ class PipelineVersionFacadeService @Autowired constructor( pipelineId = pipelineId, creator = resource.creator, createTime = resource.createTime.timestampmilli(), + updater = resource.updater, updateTime = resource.updateTime?.timestampmilli(), version = resource.version, versionName = resource.versionName ?: "", diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/SubPipelineStartUpService.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/SubPipelineStartUpService.kt index ecbf3d3784b..61e74967873 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/SubPipelineStartUpService.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/SubPipelineStartUpService.kt @@ -60,10 +60,12 @@ import com.tencent.devops.process.pojo.pipeline.SubPipelineStartUpInfo import com.tencent.devops.process.pojo.pipeline.SubPipelineStatus import com.tencent.devops.process.service.builds.PipelineBuildFacadeService import com.tencent.devops.process.service.pipeline.PipelineBuildService +import com.tencent.devops.process.utils.PIPELINE_START_SUB_RUN_MODE import com.tencent.devops.process.utils.PIPELINE_START_CHANNEL import com.tencent.devops.process.utils.PIPELINE_START_PARENT_BUILD_ID import com.tencent.devops.process.utils.PIPELINE_START_PARENT_BUILD_NUM import com.tencent.devops.process.utils.PIPELINE_START_PARENT_BUILD_TASK_ID +import com.tencent.devops.process.utils.PIPELINE_START_PARENT_EXECUTE_COUNT import com.tencent.devops.process.utils.PIPELINE_START_PARENT_PIPELINE_ID import com.tencent.devops.process.utils.PIPELINE_START_PARENT_PIPELINE_NAME import com.tencent.devops.process.utils.PIPELINE_START_PARENT_PROJECT_ID @@ -117,7 +119,8 @@ class SubPipelineStartUpService @Autowired constructor( taskId: String, runMode: String, channelCode: ChannelCode? = null, - values: Map + values: Map, + executeCount: Int? ): Result { val fixProjectId = callProjectId.ifBlank { projectId } @@ -173,7 +176,9 @@ class SubPipelineStartUpService @Autowired constructor( pipelineId = callPipelineId, channelCode = callChannelCode, parameters = startParams, - triggerUser = triggerUser + triggerUser = triggerUser, + runMode = runMode, + parentExecuteCount = executeCount ) pipelineTaskService.updateSubBuildId( projectId = projectId, @@ -185,6 +190,7 @@ class SubPipelineStartUpService @Autowired constructor( if (runMode == SYNC_RUN_MODE) { subPipelineStatusService.onStart(subBuildId) } + return Result( ProjectBuildId( id = subBuildId, @@ -204,7 +210,9 @@ class SubPipelineStartUpService @Autowired constructor( channelCode: ChannelCode, parameters: Map, isMobile: Boolean = false, - triggerUser: String? = null + triggerUser: String? = null, + runMode: String, + parentExecuteCount: Int? ): String { val readyToBuildPipelineInfo = pipelineRepositoryService.getPipelineInfo(projectId, pipelineId, channelCode) @@ -213,6 +221,9 @@ class SubPipelineStartUpService @Autowired constructor( params = arrayOf(pipelineId), errorCode = ProcessMessageCode.ERROR_PIPELINE_NOT_EXISTS ) + if (readyToBuildPipelineInfo.locked == true) { + throw ErrorCodeException(errorCode = ProcessMessageCode.ERROR_PIPELINE_LOCK) + } if (readyToBuildPipelineInfo.latestVersionStatus?.isNotReleased() == true) { throw ErrorCodeException( errorCode = ProcessMessageCode.ERROR_NO_RELEASE_PIPELINE_VERSION @@ -248,7 +259,10 @@ class SubPipelineStartUpService @Autowired constructor( val triggerContainer = model.stages[0].containers[0] as TriggerContainer // #6090 拨乱反正 - val params = buildParamCompatibilityTransformer.parseTriggerParam(triggerContainer.params, parameters) + val params = buildParamCompatibilityTransformer.parseTriggerParam( + userId = userId, projectId = projectId, pipelineId = pipelineId, + paramProperties = triggerContainer.params, paramValues = parameters + ) params[PIPELINE_START_PIPELINE_USER_ID] = BuildParameters(key = PIPELINE_START_PIPELINE_USER_ID, value = triggerUser ?: userId) @@ -260,6 +274,17 @@ class SubPipelineStartUpService @Autowired constructor( BuildParameters(key = PIPELINE_START_PARENT_PIPELINE_NAME, value = parentPipelineInfo.pipelineName) params[PIPELINE_START_PARENT_BUILD_ID] = BuildParameters(key = PIPELINE_START_PARENT_BUILD_ID, value = parentBuildId) + params[PIPELINE_START_SUB_RUN_MODE] = + BuildParameters(key = PIPELINE_START_SUB_RUN_MODE, value = runMode, readOnly = true) + // 父流水线执行次数 + parentExecuteCount?.let { + params[PIPELINE_START_PARENT_EXECUTE_COUNT] = + BuildParameters( + key = PIPELINE_START_PARENT_EXECUTE_COUNT, + value = parentExecuteCount, + readOnly = true + ) + } params[PIPELINE_START_PARENT_BUILD_NUM] = BuildParameters(key = PIPELINE_START_PARENT_BUILD_NUM, value = parentBuildInfo.buildNum) params[PIPELINE_START_PARENT_BUILD_TASK_ID] = @@ -428,14 +453,20 @@ class SubPipelineStartUpService @Autowired constructor( fun subPipelineManualStartupInfo( userId: String, projectId: String, - pipelineId: String + pipelineId: String, + includeConst: Boolean?, + includeNotRequired: Boolean? ): Result> { if (pipelineId.isBlank() || projectId.isBlank()) { return Result(ArrayList()) } val result = pipelineBuildFacadeService.buildManualStartupInfo(userId, projectId, pipelineId, ChannelCode.BS) val parameter = ArrayList() - val prop = result.properties + val prop = result.properties.filter { + val const = if (includeConst == false) { it.constant != true } else { true } + val required = if (includeNotRequired == false) { it.required } else { true } + const && required + } for (item in prop) { if (item.type == BuildFormPropertyType.MULTIPLE || item.type == BuildFormPropertyType.ENUM) { diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/builds/PipelineBuildFacadeService.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/builds/PipelineBuildFacadeService.kt index b281bb6ad0d..e416199294a 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/builds/PipelineBuildFacadeService.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/builds/PipelineBuildFacadeService.kt @@ -102,6 +102,7 @@ import com.tencent.devops.process.engine.service.record.ContainerBuildRecordServ import com.tencent.devops.process.engine.service.record.PipelineBuildRecordService import com.tencent.devops.process.engine.utils.BuildUtils import com.tencent.devops.process.engine.utils.PipelineUtils +import com.tencent.devops.process.enums.HistorySearchType import com.tencent.devops.process.jmx.api.ProcessJmxApi import com.tencent.devops.process.permission.PipelinePermissionService import com.tencent.devops.process.pojo.BuildBasicInfo @@ -254,6 +255,8 @@ class PipelineBuildFacadeService( param.value = if (param.constant == true) { param.readOnly = true param.defaultValue + } else if (!param.required) { + param.defaultValue } else if (param.defaultValue is Boolean) { realValue?.toString()?.toBoolean() } else { @@ -425,10 +428,8 @@ class PipelineBuildFacadeService( errorCode = ProcessMessageCode.ERROR_PIPELINE_NOT_EXISTS, params = arrayOf(buildId) ) - if (readyToBuildPipelineInfo.latestVersionStatus?.isNotReleased() == true) { - throw ErrorCodeException( - errorCode = ProcessMessageCode.ERROR_NO_RELEASE_PIPELINE_VERSION - ) + if (readyToBuildPipelineInfo.locked == true) { + throw ErrorCodeException(errorCode = ProcessMessageCode.ERROR_PIPELINE_LOCK) } if (!readyToBuildPipelineInfo.canManualStartup && checkManualStartup == false) { throw ErrorCodeException( @@ -575,6 +576,7 @@ class PipelineBuildFacadeService( yamlVersion = buildInfo.yamlVersion, frequencyLimit = true, handlePostFlag = false, + debug = buildInfo.debug, webHookStartParam = webHookStartParam ) } @@ -621,6 +623,9 @@ class PipelineBuildFacadeService( errorCode = ProcessMessageCode.ERROR_PIPELINE_NOT_EXISTS, params = arrayOf(pipelineId) ) + if (readyToBuildPipelineInfo.locked == true) { + throw ErrorCodeException(errorCode = ProcessMessageCode.ERROR_PIPELINE_LOCK) + } val startEpoch = System.currentTimeMillis() try { val (resource, debug) = getModelAndBuildLevel(projectId, pipelineId, version) @@ -661,10 +666,12 @@ class PipelineBuildFacadeService( logger.info("[$pipelineId] buildNo was changed to [$buildNo]") } - val paramMap = buildParamCompatibilityTransformer.parseTriggerParam(triggerContainer.params, values) + val paramMap = buildParamCompatibilityTransformer.parseTriggerParam( + userId = userId, projectId = projectId, pipelineId = pipelineId, + paramProperties = triggerContainer.params, paramValues = values + ) // 如果是PAC流水线,需要加上代码库hashId,给checkout:self使用 pipelineYamlFacadeService.buildYamlManualParamMap( - userId = userId, projectId = projectId, pipelineId = pipelineId )?.let { @@ -758,6 +765,9 @@ class PipelineBuildFacadeService( ) } val pipeline = pipelineRepositoryService.getPipelineInfo(projectId, pipelineId = pipelineId) ?: return null + if (pipeline.locked == true) { + throw ErrorCodeException(errorCode = ProcessMessageCode.ERROR_PIPELINE_LOCK) + } if (pipeline.latestVersionStatus?.isNotReleased() == true) throw ErrorCodeException( errorCode = ProcessMessageCode.ERROR_NO_RELEASE_PIPELINE_VERSION ) @@ -772,7 +782,10 @@ class PipelineBuildFacadeService( */ val triggerContainer = model.stages[0].containers[0] as TriggerContainer - val paramPamp = buildParamCompatibilityTransformer.parseTriggerParam(triggerContainer.params, parameters) + val paramPamp = buildParamCompatibilityTransformer.parseTriggerParam( + userId = userId, projectId = projectId, pipelineId = pipelineId, + paramProperties = triggerContainer.params, paramValues = parameters + ) parameters.forEach { (key, value) -> if (!paramPamp.containsKey(key)) { paramPamp[key] = BuildParameters(key = key, value = value) @@ -825,6 +838,9 @@ class PipelineBuildFacadeService( } val readyToBuildPipelineInfo = pipelineRepositoryService.getPipelineInfo(projectId, pipelineId) ?: return null + if (readyToBuildPipelineInfo.locked == true) { + throw ErrorCodeException(errorCode = ProcessMessageCode.ERROR_PIPELINE_LOCK) + } if (readyToBuildPipelineInfo.latestVersionStatus?.isNotReleased() == true) throw ErrorCodeException( errorCode = ProcessMessageCode.ERROR_NO_RELEASE_PIPELINE_VERSION ) @@ -1949,7 +1965,9 @@ class PipelineBuildFacadeService( startUser: List? = null, updateTimeDesc: Boolean? = null, archiveFlag: Boolean? = false, - debugVersion: Int? + customVersion: Int?, + triggerAlias: List?, + triggerBranch: List? ): BuildHistoryPage { val pageNotNull = page ?: 0 val pageSizeNotNull = pageSize ?: 50 @@ -1984,10 +2002,25 @@ class PipelineBuildFacadeService( permission = AuthPermission.VIEW ) } - // 如果请求的参数是草稿版本的版本号，则用该版本查询调试记录，否则正常调用普通构建 - val targetDebugVersion = debugVersion?.takeIf { - val draftVersion = pipelineRepositoryService.getDraftVersionResource(projectId, pipelineId) - draftVersion?.version == debugVersion + // 如果请求的参数是草稿版本的版本号，则返回调试记录，如果是当前正式版本则返回正式记录 + // 否则按版本查询返回空数据 + val customResource = pipelineRepositoryService.getPipelineResourceVersion( + projectId = projectId, pipelineId = pipelineId, + version = customVersion, includeDraft = true + ) + val targetDebugVersion = if (customResource?.status == VersionStatus.COMMITTING) { + customVersion + } else if (customResource?.version == pipelineInfo.version) { + null + } else { + return BuildHistoryPage( + page = pageNotNull, + pageSize = limit, + count = 0, + records = emptyList(), + hasDownloadPermission = false, + pipelineVersion = pipelineInfo.version + ) } val newTotalCount = pipelineRuntimeService.getPipelineBuildHistoryCount( @@ -2014,7 +2047,9 @@ class PipelineBuildFacadeService( buildMsg = buildMsg, startUser = startUser, queryDslContext = queryDslContext, - debugVersion = targetDebugVersion + debugVersion = targetDebugVersion, + triggerAlias = triggerAlias, + triggerBranch = triggerBranch ) val newHistoryBuilds = pipelineRuntimeService.listPipelineBuildHistory( @@ -2044,7 +2079,9 @@ class PipelineBuildFacadeService( startUser = startUser, updateTimeDesc = updateTimeDesc, queryDslContext = queryDslContext, - debugVersion = targetDebugVersion + debugVersion = targetDebugVersion, + triggerAlias = triggerAlias, + triggerBranch = triggerBranch ) val buildHistories = mutableListOf() buildHistories.addAll(newHistoryBuilds) @@ -2126,7 +2163,9 @@ class PipelineBuildFacadeService( userId: String, projectId: String, pipelineId: String, - debugVersion: Int? + debugVersion: Int?, + search: String?, + type: HistorySearchType? ): List { pipelinePermissionService.validPipelinePermission( userId = userId, @@ -2144,7 +2183,13 @@ class PipelineBuildFacadeService( val draftVersion = pipelineRepositoryService.getDraftVersionResource(projectId, pipelineId) draftVersion?.version == debugVersion } - return pipelineRuntimeService.getHistoryConditionRepo(projectId, pipelineId, targetDebugVersion) + return pipelineRuntimeService.getHistoryConditionRepo( + projectId = projectId, + pipelineId = pipelineId, + debugVersion = targetDebugVersion, + search = search, + type = type + ) } fun getHistoryConditionBranch( @@ -2152,7 +2197,9 @@ class PipelineBuildFacadeService( projectId: String, pipelineId: String, alias: List?, - debugVersion: Int? + debugVersion: Int?, + search: String?, + type: HistorySearchType? ): List { pipelinePermissionService.validPipelinePermission( userId = userId, @@ -2171,7 +2218,12 @@ class PipelineBuildFacadeService( draftVersion?.version == debugVersion } return pipelineRuntimeService.getHistoryConditionBranch( - projectId, pipelineId, alias, targetDebugVersion + projectId = projectId, + pipelineId = pipelineId, + aliasList = alias, + debugVersion = targetDebugVersion, + search = search, + type = type ) } @@ -2186,7 +2238,8 @@ class PipelineBuildFacadeService( buildId = buildId, projectId = build.projectId, pipelineId = build.pipelineId, - pipelineVersion = build.version + pipelineVersion = build.version, + status = build.status ) } @@ -2202,6 +2255,7 @@ class PipelineBuildFacadeService( projectId: String, pipelineId: String, buildNum: Int, + buildId: String?, channelCode: ChannelCode ): BuildHistory? { val statusSet = mutableSetOf() @@ -2214,11 +2268,15 @@ class PipelineBuildFacadeService( } } } + val buildInfo = buildId?.let { + pipelineRuntimeService.getBuildInfo(projectId, pipelineId, buildId) + } val buildHistory = pipelineRuntimeService.getBuildHistoryByBuildNum( projectId = projectId, pipelineId = pipelineId, buildNum = buildNum, - statusSet = statusSet + statusSet = statusSet, + debug = buildInfo?.debug ) logger.info("[$pipelineId]|buildHistory=$buildHistory") return buildHistory @@ -2227,13 +2285,18 @@ class PipelineBuildFacadeService( fun getLatestSuccessBuild( projectId: String, pipelineId: String, + buildId: String?, channelCode: ChannelCode ): BuildHistory? { + val buildInfo = buildId?.let { + pipelineRuntimeService.getBuildInfo(projectId, pipelineId, buildId) + } val buildHistory = pipelineRuntimeService.getBuildHistoryByBuildNum( projectId = projectId, pipelineId = pipelineId, buildNum = -1, - statusSet = setOf(BuildStatus.SUCCEED) + statusSet = setOf(BuildStatus.SUCCEED), + debug = buildInfo?.debug ) logger.info("[$pipelineId]|buildHistory=$buildHistory") return buildHistory diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineSettingFacadeService.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineSettingFacadeService.kt index 3c3d6881c5f..9fe57d404e0 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineSettingFacadeService.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/pipeline/PipelineSettingFacadeService.kt @@ -60,6 +60,7 @@ import com.tencent.devops.process.pojo.config.TaskCommonSettingConfig import com.tencent.devops.process.pojo.setting.JobCommonSetting import com.tencent.devops.process.pojo.setting.PipelineCommonSetting import com.tencent.devops.common.pipeline.pojo.setting.PipelineSetting +import com.tencent.devops.process.permission.template.PipelineTemplatePermissionService import com.tencent.devops.process.pojo.setting.StageCommonSetting import com.tencent.devops.process.pojo.setting.PipelineSettingVersion import com.tencent.devops.process.pojo.setting.TaskCommonSetting @@ -87,7 +88,8 @@ class PipelineSettingFacadeService @Autowired constructor( private val taskCommonSettingConfig: TaskCommonSettingConfig, private val auditService: AuditService, private val modelCheckPlugin: ModelCheckPlugin, - private val pipelineEventDispatcher: PipelineEventDispatcher + private val pipelineEventDispatcher: PipelineEventDispatcher, + private val pipelineTemplatePermissionService: PipelineTemplatePermissionService ) { private val logger = LoggerFactory.getLogger(PipelineSettingFacadeService::class.java) @@ -178,11 +180,20 @@ class PipelineSettingFacadeService @Autowired constructor( ) if (checkPermission) { - pipelinePermissionService.modifyResource( - projectId = setting.projectId, - pipelineId = setting.pipelineId, - pipelineName = setting.pipelineName - ) + if (isTemplate) { + pipelineTemplatePermissionService.modifyResource( + userId = userId, + projectId = projectId, + templateId = setting.pipelineId, + templateName = setting.pipelineName + ) + } else { + pipelinePermissionService.modifyResource( + projectId = setting.projectId, + pipelineId = setting.pipelineId, + pipelineName = setting.pipelineName + ) + } } } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/template/TemplateFacadeService.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/template/TemplateFacadeService.kt index a38199785ab..e7bb18cd178 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/template/TemplateFacadeService.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/template/TemplateFacadeService.kt @@ -57,7 +57,6 @@ import com.tencent.devops.common.pipeline.enums.PipelineInstanceTypeEnum import com.tencent.devops.common.pipeline.extend.ModelCheckPlugin import com.tencent.devops.common.pipeline.pojo.BuildFormProperty import com.tencent.devops.common.pipeline.pojo.BuildNo -import com.tencent.devops.common.pipeline.pojo.element.Element import com.tencent.devops.common.pipeline.pojo.element.agent.CodeGitElement import com.tencent.devops.common.pipeline.pojo.element.agent.CodeSvnElement import com.tencent.devops.common.pipeline.pojo.element.agent.GithubElement @@ -129,6 +128,7 @@ import com.tencent.devops.process.service.pipeline.PipelineSettingFacadeService import com.tencent.devops.process.util.TempNotifyTemplateUtils import com.tencent.devops.process.utils.KEY_PIPELINE_ID import com.tencent.devops.process.utils.KEY_TEMPLATE_ID +import com.tencent.devops.process.utils.PipelineVersionUtils.differ import com.tencent.devops.project.api.service.ServiceAllocIdResource import com.tencent.devops.repository.api.ServiceRepositoryResource import com.tencent.devops.store.api.common.ServiceStoreResource @@ -149,8 +149,6 @@ import java.text.MessageFormat import java.time.LocalDateTime import javax.ws.rs.NotFoundException import javax.ws.rs.core.Response -import kotlin.reflect.full.declaredMemberProperties -import kotlin.reflect.jvm.isAccessible @Suppress("ALL") @Service @@ -371,7 +369,7 @@ class TemplateFacadeService @Autowired constructor( statusCode = Response.Status.NOT_FOUND.statusCode, errorCode = ProcessMessageCode.ERROR_PIPELINE_MODEL_NOT_EXISTS ) - val templateModel: Model = objectMapper.readValue(template) + val templateModel: Model = PipelineUtils.fixedTemplateParam(objectMapper.readValue(template)) checkTemplateAtomsForExplicitVersion(templateModel, userId) val templateId = UUIDUtil.generate() dslContext.transaction { configuration -> @@ -384,7 +382,7 @@ class TemplateFacadeService @Autowired constructor( templateName = saveAsTemplateReq.templateName, versionName = INIT_TEMPLATE_NAME, userId = userId, - template = template, + template = JsonUtil.toJson(templateModel, formatted = false), storeFlag = false, version = client.get(ServiceAllocIdResource::class).generateSegmentId(TEMPLATE_BIZ_TAG_NAME).data, desc = null @@ -1435,8 +1433,7 @@ class TemplateFacadeService @Autowired constructor( container.elements.forEach e@{ element -> v2Container.elements.forEach { v2Element -> if (element.id == v2Element.id) { - val modify = elementModify(element, v2Element) - if (modify) { + if (element.differ(v2Element)) { element.templateModify = true v2Element.templateModify = true } @@ -1451,42 +1448,6 @@ class TemplateFacadeService @Autowired constructor( return objectMapper.readValue(templateModelStr) } - fun elementModify(e1: Element, e2: Element): Boolean { - if (e1::class != e2::class) { - return true - } - - val v1Properties = e1.javaClass.kotlin.declaredMemberProperties - val v2Properties = e2.javaClass.kotlin.declaredMemberProperties - if (v1Properties.size != v2Properties.size) { - return true - } - - val v1Map = v1Properties.associate { - it.isAccessible = true - it.name to it.get(e1) - } - - val v2Map = v2Properties.associate { - it.isAccessible = true - it.name to it.get(e2) - } - - if (v1Map.size != v2Map.size) { - return true - } - - for ((key, value) in v1Map) { - if (!v2Map.containsKey(key)) { - return true - } - if (v2Map[key] != value) { - return true - } - } - return false - } - private fun getContainers(model: Model): Map { val containers = HashMap() @@ -2421,6 +2382,9 @@ class TemplateFacadeService @Autowired constructor( stage.containers.forEach { container -> if (container is TriggerContainer) { container.params = PipelineUtils.cleanOptions(params = container.params) + container.templateParams = container.templateParams?.let { + PipelineUtils.cleanOptions(params = it) + } } if (container.containerId.isNullOrBlank()) { container.containerId = container.id diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/webhook/PipelineBuildWebhookService.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/webhook/PipelineBuildWebhookService.kt index 9b457bc8839..f484b3ec6e0 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/webhook/PipelineBuildWebhookService.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/service/webhook/PipelineBuildWebhookService.kt @@ -38,10 +38,10 @@ import com.tencent.devops.common.log.utils.BuildLogPrinter import com.tencent.devops.common.pipeline.container.TriggerContainer import com.tencent.devops.common.pipeline.enums.ChannelCode import com.tencent.devops.common.pipeline.enums.StartType -import com.tencent.devops.common.pipeline.enums.VersionStatus import com.tencent.devops.common.pipeline.pojo.BuildParameters import com.tencent.devops.common.pipeline.pojo.element.trigger.WebHookTriggerElement import com.tencent.devops.common.pipeline.utils.PIPELINE_PAC_REPO_HASH_ID +import com.tencent.devops.common.service.prometheus.BkTimed import com.tencent.devops.common.webhook.pojo.code.PIPELINE_START_WEBHOOK_USER_ID import com.tencent.devops.common.webhook.service.code.loader.WebhookElementParamsRegistrar import com.tencent.devops.common.webhook.service.code.loader.WebhookStartParamsRegistrar @@ -78,6 +78,7 @@ import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service import java.time.LocalDate +import javax.ws.rs.core.Response @Suppress("ALL") @Service @@ -199,6 +200,7 @@ class PipelineBuildWebhookService @Autowired constructor( } } + @BkTimed private fun webhookTriggerPipelineBuild( projectId: String, pipelineId: String, @@ -307,6 +309,10 @@ class PipelineBuildWebhookService @Autowired constructor( } } catch (ignore: Exception) { logger.warn("$pipelineId|webhook trigger|(${element.name})|repo(${matcher.getRepoName()})", ignore) + builder.eventSource(eventSource = repo.repoHashId!!) + builder.status(PipelineTriggerStatus.FAILED.name) + .reason(PipelineTriggerReason.TRIGGER_FAILED.name) + .reasonDetail(PipelineTriggerFailedMsg(ignore.message ?: "")) } return true } else { @@ -489,11 +495,18 @@ class PipelineBuildWebhookService @Autowired constructor( val repoName = webhookCommit.repoName val pipelineInfo = pipelineRepositoryService.getPipelineInfo(projectId, pipelineId) - ?: throw IllegalArgumentException("Pipeline($pipelineId) not found") - // 代码库触发支持仅有分支版本的情况 - if (pipelineInfo.latestVersionStatus == VersionStatus.COMMITTING) throw ErrorCodeException( - errorCode = ProcessMessageCode.ERROR_NO_RELEASE_PIPELINE_VERSION - ) + ?: throw ErrorCodeException( + statusCode = Response.Status.NOT_FOUND.statusCode, + errorCode = ProcessMessageCode.ERROR_PIPELINE_NOT_EXISTS, + params = arrayOf(pipelineId) + ) + if (pipelineInfo.locked == true) { + throw ErrorCodeException(errorCode = ProcessMessageCode.ERROR_PIPELINE_LOCK) + } + // 代码库触发支持仅有分支版本的情况，如果仅有草稿不需要在这里拦截 +// if (pipelineInfo.latestVersionStatus == VersionStatus.COMMITTING) throw ErrorCodeException( +// errorCode = ProcessMessageCode.ERROR_NO_RELEASE_PIPELINE_VERSION +// ) val version = webhookCommit.version ?: pipelineInfo.version checkPermission(pipelineInfo.lastModifyUser, projectId = projectId, pipelineId = pipelineId) @@ -576,7 +589,7 @@ class PipelineBuildWebhookService @Autowired constructor( return buildId } catch (ignore: Exception) { logger.warn("[$pipelineId]| webhook trigger fail to start repo($repoName): ${ignore.message}", ignore) - return null + throw ignore } finally { logger.info("$pipelineId|WEBHOOK_TRIGGER|repo=$repoName|time=${System.currentTimeMillis() - startEpoch}") } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/trigger/PipelineTriggerEventAspect.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/trigger/PipelineTriggerEventAspect.kt index 6a5ba0066c1..ca529f67a0e 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/trigger/PipelineTriggerEventAspect.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/trigger/PipelineTriggerEventAspect.kt @@ -47,6 +47,7 @@ import com.tencent.devops.process.pojo.trigger.PipelineTriggerDetailBuilder import com.tencent.devops.process.pojo.trigger.PipelineTriggerEvent import com.tencent.devops.process.pojo.trigger.PipelineTriggerEventBuilder import com.tencent.devops.process.pojo.trigger.PipelineTriggerFailedErrorCode +import com.tencent.devops.process.pojo.trigger.PipelineTriggerFailedMatch import com.tencent.devops.process.pojo.trigger.PipelineTriggerFailedMsg import com.tencent.devops.process.pojo.trigger.PipelineTriggerReason import com.tencent.devops.process.pojo.trigger.PipelineTriggerReasonDetail @@ -356,7 +357,12 @@ class PipelineTriggerEventAspect( reasonDetail != null -> { triggerDetailBuilder.status(PipelineTriggerStatus.FAILED.name) - triggerDetailBuilder.reason(PipelineTriggerReason.TRIGGER_FAILED.name) + if (reasonDetail is PipelineTriggerFailedMatch) { + triggerDetailBuilder.reason(PipelineTriggerReason.TRIGGER_NOT_MATCH.name) + } else { + triggerDetailBuilder.reason(PipelineTriggerReason.TRIGGER_FAILED.name) + } + triggerDetailBuilder.reasonDetail(reasonDetail) } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/webhook/TriggerBuildParamUtils.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/webhook/TriggerBuildParamUtils.kt index dcbfa21b704..98069c3f7b4 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/webhook/TriggerBuildParamUtils.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/webhook/TriggerBuildParamUtils.kt @@ -89,6 +89,7 @@ import com.tencent.devops.process.constant.PipelineBuildParamKey.CI_PIPELINE_NAM import com.tencent.devops.process.constant.PipelineBuildParamKey.CI_PIPELINE_VERSION import com.tencent.devops.process.constant.PipelineBuildParamKey.CI_PROJECT_ID import com.tencent.devops.process.constant.PipelineBuildParamKey.CI_PROJECT_NAME +import com.tencent.devops.process.constant.PipelineBuildParamKey.CI_REMARK import com.tencent.devops.process.constant.PipelineBuildParamKey.CI_REPO import com.tencent.devops.process.constant.PipelineBuildParamKey.CI_REPO_ALIAS_NAME import com.tencent.devops.process.constant.PipelineBuildParamKey.CI_REPO_GROUP @@ -150,7 +151,8 @@ object TriggerBuildParamUtils { CI_BUILD_START_TYPE, CI_WORKSPACE, CI_FAILED_TASKNAMES, - CI_FAILED_TASKS + CI_FAILED_TASKS, + CI_REMARK ).sortedBy { it }.map { diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/webhook/WebHookTriggerElementBizPlugin.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/webhook/WebHookTriggerElementBizPlugin.kt index 7a874681242..f778cd7d6cf 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/webhook/WebHookTriggerElementBizPlugin.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/webhook/WebHookTriggerElementBizPlugin.kt @@ -42,6 +42,7 @@ import com.tencent.devops.common.pipeline.pojo.element.trigger.WebHookTriggerEle import com.tencent.devops.process.engine.service.PipelineWebhookService import com.tencent.devops.process.plugin.ElementBizPlugin import com.tencent.devops.process.plugin.annotation.ElementBiz +import com.tencent.devops.process.pojo.pipeline.PipelineYamlVo abstract class WebHookTriggerElementBizPlugin constructor( private val pipelineWebhookService: PipelineWebhookService @@ -54,7 +55,8 @@ abstract class WebHookTriggerElementBizPlugin constru userId: String, channelCode: ChannelCode, create: Boolean, - container: Container + container: Container, + yamlInfo: PipelineYamlVo? ) = Unit override fun beforeDelete(element: T, param: BeforeDeleteParam) { diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/webhook/WebhookRequestService.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/webhook/WebhookRequestService.kt index 9aca356668b..1c6a622f6b3 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/webhook/WebhookRequestService.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/webhook/WebhookRequestService.kt @@ -79,7 +79,13 @@ class WebhookRequestService( externalId = matcher.getExternalId(), eventType = matcher.getEventType().name, triggerUser = matcher.getUsername(), - eventMessage = matcher.getMessage() ?: "", + eventMessage = matcher.getMessage()?.let { + if (it.length >= 128) { + it.substring(0, 128) + } else { + it + } + } ?: "", repositoryType = scmType.name, requestHeader = request.headers, requestParam = request.queryParams, diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/PipelineYamlFacadeService.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/PipelineYamlFacadeService.kt index a298b665c17..aea246c6a9c 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/PipelineYamlFacadeService.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/PipelineYamlFacadeService.kt @@ -172,7 +172,6 @@ class PipelineYamlFacadeService @Autowired constructor( val repository = client.get(ServiceRepositoryPacResource::class).getPacRepository( externalId = externalId, scmType = scmType ).data ?: run { - logger.info("pipeline yaml trigger|repository not enable pac|$externalId|$scmType") return } val setting = PacRepoSetting(repository = repository) @@ -291,6 +290,44 @@ class PipelineYamlFacadeService @Autowired constructor( )[pipelineId] ?: false } + fun checkPushParam( + userId: String, + projectId: String, + pipelineId: String, + version: Int, + versionName: String?, + repoHashId: String, + scmType: ScmType, + filePath: String, + content: String, + targetAction: CodeTargetAction + ) { + logger.info("check push yaml file|$userId|$projectId|$pipelineId|$repoHashId|$scmType|$version|$versionName") + val repository = client.get(ServiceRepositoryResource::class).get( + projectId = projectId, + repositoryId = repoHashId, + repositoryType = RepositoryType.ID + ).data ?: throw ErrorCodeException( + errorCode = ProcessMessageCode.GIT_NOT_FOUND, + params = arrayOf(repoHashId) + ) + checkPushParam(content, repoHashId, filePath, targetAction, versionName, projectId, pipelineId) + val setting = PacRepoSetting(repository = repository) + val event = PipelineYamlManualEvent( + userId = userId, + projectId = projectId, + repoHashId = repoHashId, + scmType = scmType + ) + val action = eventActionFactory.loadManualEvent(setting = setting, event = event) + if (!action.checkPushPermission()) { + throw ErrorCodeException( + errorCode = ProcessMessageCode.ERROR_YAML_PUSH_NO_REPO_PERMISSION, + params = arrayOf(userId, repository.url) + ) + } + } + fun pushYamlFile( userId: String, projectId: String, @@ -415,28 +452,14 @@ class PipelineYamlFacadeService @Autowired constructor( /** * 构建yaml流水线触发变量 */ - fun buildYamlManualParamMap(userId: String, projectId: String, pipelineId: String): Map? { + fun buildYamlManualParamMap(projectId: String, pipelineId: String): Map? { val pipelineYamlInfo = pipelineYamlInfoDao.get( dslContext = dslContext, projectId = projectId, pipelineId = pipelineId ) ?: return null - val repoHashId = pipelineYamlInfo.repoHashId - val repository = client.get(ServiceRepositoryResource::class).get( - projectId = projectId, - repositoryId = repoHashId, - repositoryType = RepositoryType.ID - ).data ?: return null - val setting = PacRepoSetting(repository = repository) - val event = PipelineYamlManualEvent( - userId = userId, - projectId = projectId, - repoHashId = repoHashId, - scmType = repository.getScmType() - ) - val action = eventActionFactory.loadManualEvent(setting = setting, event = event) return mutableMapOf( - BK_REPO_WEBHOOK_HASH_ID to BuildParameters(BK_REPO_WEBHOOK_HASH_ID, repoHashId), + BK_REPO_WEBHOOK_HASH_ID to BuildParameters(BK_REPO_WEBHOOK_HASH_ID, pipelineYamlInfo.repoHashId), PIPELINE_WEBHOOK_BRANCH to BuildParameters( - PIPELINE_WEBHOOK_BRANCH, action.data.context.defaultBranch ?: "" + PIPELINE_WEBHOOK_BRANCH, pipelineYamlInfo.defaultBranch ?: "" ) ) } diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/PipelineYamlRepositoryService.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/PipelineYamlRepositoryService.kt index b9b7ee801a7..87554192890 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/PipelineYamlRepositoryService.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/PipelineYamlRepositoryService.kt @@ -41,6 +41,7 @@ import com.tencent.devops.common.pipeline.utils.RepositoryConfigUtils import com.tencent.devops.common.redis.RedisOperation import com.tencent.devops.process.engine.service.PipelineRepositoryService import com.tencent.devops.process.engine.service.PipelineWebhookService +import com.tencent.devops.process.pojo.pipeline.PipelineYamlVo import com.tencent.devops.process.pojo.pipeline.enums.PipelineYamlStatus import com.tencent.devops.process.pojo.webhook.PipelineWebhookVersion import com.tencent.devops.process.service.PipelineInfoFacadeService @@ -186,7 +187,7 @@ class PipelineYamlRepositoryService @Autowired constructor( blobId: String, defaultBranch: String ): Boolean { - val pipelineYamlVersion = pipelineYamlService.getLatestVersionByRef( + val pipelineYamlVersion = pipelineYamlService.getLatestVersion( projectId = projectId, repoHashId = repoHashId, filePath = filePath, @@ -195,7 +196,7 @@ class PipelineYamlRepositoryService @Autowired constructor( ) return if (pipelineYamlVersion == null) { if (defaultBranch != ref) { - pipelineYamlService.getLatestVersionByRef( + pipelineYamlService.getLatestVersion( projectId = projectId, repoHashId = repoHashId, filePath = filePath, @@ -231,6 +232,10 @@ class PipelineYamlRepositoryService @Autowired constructor( DateTimeUtil.stringToLocalDateTime(it) } ?: LocalDateTime.now() val ref = GitActionCommon.getRealRef(action = action, branch = branch) + val yamlInfo = PipelineYamlVo( + repoHashId = repoHashId, + filePath = yamlFile.yamlPath + ) val deployPipelineResult = pipelineInfoFacadeService.createYamlPipeline( userId = action.data.setting.enableUser, projectId = projectId, @@ -241,7 +246,8 @@ class PipelineYamlRepositoryService @Autowired constructor( description = action.data.eventCommon.commit.commitMsg, aspects = PipelineTransferAspectLoader.initByDefaultTriggerOn(defaultRepo = { action.data.setting.aliasName - }) + }), + yamlInfo = yamlInfo ) val pipelineId = deployPipelineResult.pipelineId val version = deployPipelineResult.version @@ -296,7 +302,12 @@ class PipelineYamlRepositoryService @Autowired constructor( } ?: LocalDateTime.now() // 如果是fork仓库,ref应该加上fork库的namespace val ref = GitActionCommon.getRealRef(action = action, branch = branch) + val repoHashId = action.data.setting.repoHashId + val yamlInfo = PipelineYamlVo( + repoHashId = repoHashId, + filePath = yamlFile.yamlPath + ) val deployPipelineResult = pipelineInfoFacadeService.updateYamlPipeline( userId = action.data.setting.enableUser, projectId = projectId, @@ -308,10 +319,11 @@ class PipelineYamlRepositoryService @Autowired constructor( description = action.data.eventCommon.commit.commitMsg, aspects = PipelineTransferAspectLoader.initByDefaultTriggerOn(defaultRepo = { action.data.setting.aliasName - }) + }), + yamlInfo = yamlInfo ) val version = deployPipelineResult.version - val repoHashId = action.data.setting.repoHashId + val webhooks = getWebhooks( projectId = projectId, pipelineId = pipelineId, version = version, repoHashId = repoHashId ) @@ -353,10 +365,17 @@ class PipelineYamlRepositoryService @Autowired constructor( val defaultBranch = action.data.context.defaultBranch val ref = yamlFile.ref logger.info("deleteYamlPipeline|$userId|$projectId|$repoHashId|yamlFile:$yamlFile") + if (ref.isNullOrBlank()) { + return + } + val lock = PipelineYamlTriggerLock( + redisOperation = redisOperation, + projectId = projectId, + repoHashId = repoHashId, + filePath = filePath + ) try { - if (ref.isNullOrBlank()) { - return - } + lock.lock() pipelineYamlService.deleteBranchFile( projectId = projectId, repoHashId = repoHashId, @@ -368,8 +387,24 @@ class PipelineYamlRepositoryService @Autowired constructor( repoHashId = repoHashId, filePath = filePath ) ?: return - // 非默认分支删除yaml,需要将分支版本置为无效 - if (ref != defaultBranch) { + val needUnbindYamlPipeline = needUnbindYamlPipeline( + projectId = projectId, + repoHashId = repoHashId, + filePath = filePath, + ref = ref, + defaultBranch = defaultBranch + ) + // 只有删除分支或者删除文件时才能解绑 + if (releaseBranch == false && needUnbindYamlPipeline) { + unBindYamlPipeline( + userId = userId, + projectId = projectId, + pipelineId = pipelineYamlInfo.pipelineId, + repoHashId = repoHashId, + filePath = yamlFile.yamlPath, + refreshView = true + ) + } else { pipelineYamlService.updateBranchAction( projectId = projectId, repoHashId = repoHashId, @@ -393,17 +428,48 @@ class PipelineYamlRepositoryService @Autowired constructor( defaultBranch = defaultBranch ) } - } else { - pipelineYamlService.updatePipelineYamlStatus( - projectId = projectId, - repoHashId = repoHashId, - filePath = filePath, - status = PipelineYamlStatus.DELETED.name - ) } } catch (ignored: Exception) { logger.warn("Failed to delete pipeline yaml|$projectId|${action.format()}", ignored) throw ignored + } finally { + lock.unlock() + } + } + + /** + * 是否需要解绑PAC流水线 + * 1. 默认分支yaml删除,直接解绑 + * 2. 非默认分支删除 + * - 默认分支存在yaml文件,不能解绑 + * - 默认分支不存在yaml文件,当前流水线最新版本对应的分支与当前分支相同,可以解绑 + */ + private fun needUnbindYamlPipeline( + projectId: String, + repoHashId: String, + filePath: String, + ref: String, + defaultBranch: String? + ): Boolean { + return when { + defaultBranch == null -> false + ref == defaultBranch -> true + else -> { + // 如果存在稳定版本,说明yaml在默认分支存在,那么在非默认分支删除,不能关闭PAC, + // 否则判断当前流水线最新版本是不是当前分支创建,是-删除,不是-不删 + pipelineYamlService.getLatestVersion( + projectId = projectId, + repoHashId = repoHashId, + filePath = filePath, + ref = defaultBranch + )?.let { false } ?: run { + pipelineYamlService.getLatestVersion( + projectId = projectId, + repoHashId = repoHashId, + filePath = filePath + )?.ref == ref + } + } } } @@ -547,17 +613,13 @@ class PipelineYamlRepositoryService @Autowired constructor( val yamlPipelines = pipelineYamlService.getAllYamlPipeline(projectId = projectId, repoHashId = repoHashId) yamlPipelines.forEach { pipelineYamlInfo -> // 解绑yaml关联的流水线 - pipelineInfoFacadeService.updateYamlPipelineSetting( + unBindYamlPipeline( userId = userId, projectId = projectId, pipelineId = pipelineYamlInfo.pipelineId, - pipelineAsCodeSettings = PipelineAsCodeSettings(enable = false) - ) - pipelineYamlService.deleteYamlPipeline( - userId = userId, - projectId = projectId, repoHashId = repoHashId, - filePath = pipelineYamlInfo.filePath + filePath = pipelineYamlInfo.filePath, + refreshView = false ) } // 删除流水线组 @@ -579,6 +641,47 @@ class PipelineYamlRepositoryService @Autowired constructor( pipelineYamlSyncService.delete(projectId = projectId, repoHashId = repoHashId) } + /** + * 解绑流水线PAC,有两种情况会解绑流水线PAC + * 1. 代码库直接关闭PAC，关闭前需要删除所有默认分支yaml文件 + * 2. 默认分支删除yaml文件 + * 3. 删除非默认分支yaml文件，并且流水线没有稳定版本 + * + * @param refreshView 是否刷新流水线组 + */ + private fun unBindYamlPipeline( + userId: String, + projectId: String, + pipelineId: String, + repoHashId: String, + filePath: String, + refreshView: Boolean + ) { + // 解绑yaml关联的流水线 + pipelineInfoFacadeService.updateYamlPipelineSetting( + userId = userId, + projectId = projectId, + pipelineId = pipelineId, + pipelineAsCodeSettings = PipelineAsCodeSettings(enable = false) + ) + pipelineYamlService.deleteYamlPipeline( + userId = userId, + projectId = projectId, + repoHashId = repoHashId, + filePath = filePath + ) + if (refreshView) { + val pipelineInfo = pipelineRepositoryService.getPipelineInfo(projectId, pipelineId) ?: return + pipelineViewGroupService.updateGroupAfterPipelineUpdate( + projectId = projectId, + pipelineId = pipelineId, + pipelineName = pipelineInfo.pipelineName, + creator = userId, + userId = userId + ) + } + } + /** * TODO 需优化 * 本来应该在com.tencent.devops.process.engine.service.PipelineWebhookService.addWebhook处理, diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/PipelineYamlService.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/PipelineYamlService.kt index 3c62ba9d1dd..668c21e0146 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/PipelineYamlService.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/PipelineYamlService.kt @@ -313,11 +313,11 @@ class PipelineYamlService( /** * 获取当前分支或blob_id对应的最新的版本 */ - fun getLatestVersionByRef( + fun getLatestVersion( projectId: String, repoHashId: String, filePath: String, - ref: String, + ref: String? = null, blobId: String? = null, branchAction: String? = null ): PipelineYamlVersion? { diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/actions/internal/PipelineYamlManualAction.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/actions/internal/PipelineYamlManualAction.kt index 1565c5923fa..b798fd5d9a7 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/actions/internal/PipelineYamlManualAction.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/actions/internal/PipelineYamlManualAction.kt @@ -147,6 +147,14 @@ class PipelineYamlManualAction : BaseAction { override fun getChangeSet(): Set? = null + fun checkPushPermission(): Boolean { + return api.checkPushPermission( + userId = event().userId, + cred = this.getGitCred(), + gitProjectId = getGitProjectIdOrName() + ) + } + fun pushYamlFile( pipelineId: String, pipelineName: String, diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/actions/tgit/TGitMrActionGit.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/actions/tgit/TGitMrActionGit.kt index 715a0af818f..dc6333859f6 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/actions/tgit/TGitMrActionGit.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/actions/tgit/TGitMrActionGit.kt @@ -446,17 +446,6 @@ class TGitMrActionGit( ) ) } - // 源分支有，目标分支有，变更列表无，以目标分支为主，不需要校验版本 - source in targetList && source !in changeSet -> { - result.add( - YamlPathListEntry( - source, - CheckType.NO_NEED_CHECK, - targetRef, - blobId - ) - ) - } } } @@ -480,6 +469,17 @@ class TGitMrActionGit( ) ) } + // 源分支有，目标分支有，变更列表无，以目标分支为主，不需要校验版本 + target in sourceList && target !in changeSet -> { + result.add( + YamlPathListEntry( + target, + CheckType.NO_NEED_CHECK, + targetRef, + blobId + ) + ) + } } } return result diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/exception/hanlder/YamlTriggerExceptionUtil.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/exception/hanlder/YamlTriggerExceptionUtil.kt index fe4d63c957b..fc94adb36af 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/exception/hanlder/YamlTriggerExceptionUtil.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/exception/hanlder/YamlTriggerExceptionUtil.kt @@ -34,22 +34,21 @@ import com.tencent.devops.process.pojo.trigger.PipelineTriggerFailedErrorCode import com.tencent.devops.process.pojo.trigger.PipelineTriggerFailedMsg import com.tencent.devops.process.pojo.trigger.PipelineTriggerReason import com.tencent.devops.process.pojo.trigger.PipelineTriggerReasonDetail -import com.tencent.devops.process.pojo.trigger.PipelineTriggerStatus import com.tencent.devops.process.yaml.exception.YamlTriggerException object YamlTriggerExceptionUtil { fun getReasonDetail(exception: Exception): Pair { return when (exception) { is YamlTriggerException -> Pair( - PipelineTriggerStatus.FAILED.name, + PipelineTriggerReason.TRIGGER_FAILED.name, PipelineTriggerFailedErrorCode(errorCode = exception.errorCode, params = exception.params?.toList()) ) is ErrorCodeException -> Pair( - PipelineTriggerStatus.FAILED.name, + PipelineTriggerReason.TRIGGER_FAILED.name, PipelineTriggerFailedErrorCode(errorCode = exception.errorCode, params = exception.params?.toList()) ) is NotFoundException -> Pair( - PipelineTriggerStatus.FAILED.name, + PipelineTriggerReason.TRIGGER_FAILED.name, PipelineTriggerFailedMsg(exception.message ?: PipelineTriggerReason.UNKNOWN_ERROR.detail) ) else -> Pair( diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/git/service/PacGitApiService.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/git/service/PacGitApiService.kt index 8df6265667b..34f4632448e 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/git/service/PacGitApiService.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/git/service/PacGitApiService.kt @@ -137,6 +137,8 @@ interface PacGitApiService { retry: ApiRequestRetryInfo ): PacGitFileInfo? + fun checkPushPermission(userId: String, cred: PacGitCred, gitProjectId: String): Boolean + /** * 提交yaml文件 */ diff --git a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/git/service/TGitApiService.kt b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/git/service/TGitApiService.kt index 27085420db0..6286a6c0af4 100644 --- a/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/git/service/TGitApiService.kt +++ b/src/backend/ci/core/process/biz-process/src/main/kotlin/com/tencent/devops/process/yaml/git/service/TGitApiService.kt @@ -53,6 +53,7 @@ import com.tencent.devops.process.yaml.git.pojo.tgit.TGitTreeFileInfo import com.tencent.devops.process.yaml.git.service.PacApiUtil.doRetryFun import com.tencent.devops.repository.api.ServiceOauthResource import com.tencent.devops.repository.api.scm.ServiceGitResource +import com.tencent.devops.repository.pojo.enums.GitAccessLevelEnum import com.tencent.devops.repository.pojo.enums.RepoAuthType import com.tencent.devops.repository.pojo.enums.TokenTypeEnum import com.tencent.devops.repository.pojo.git.GitOperationFile @@ -286,6 +287,21 @@ class TGitApiService @Autowired constructor( }?.let { TGitFileInfo(content = it.content ?: "", blobId = it.blobId) } } + override fun checkPushPermission(userId: String, cred: PacGitCred, gitProjectId: String): Boolean { + return try { + val member = client.get(ServiceGitResource::class).getProjectMembersAll( + gitProjectId = gitProjectId, + search = userId, + tokenType = TokenTypeEnum.OAUTH, + token = cred.toToken() + ).data?.find { it.username == userId } + member?.let { it.accessLevel >= GitAccessLevelEnum.DEVELOPER.level } ?: false + } catch (ignored: Throwable) { + logger.warn("Failed to check push permission|$userId|$gitProjectId") + false + } + } + @Suppress("ComplexMethod") override fun pushYamlFile( userId: String, diff --git a/src/backend/ci/core/process/biz-process/src/test/kotlin/com/tencent/devops/process/TestBase.kt b/src/backend/ci/core/process/biz-process/src/test/kotlin/com/tencent/devops/process/TestBase.kt index e7e5b83f05e..330268e0678 100644 --- a/src/backend/ci/core/process/biz-process/src/test/kotlin/com/tencent/devops/process/TestBase.kt +++ b/src/backend/ci/core/process/biz-process/src/test/kotlin/com/tencent/devops/process/TestBase.kt @@ -219,6 +219,7 @@ open class TestBase { buildId = buildId, stageId = stageId, containerId = id?.toString() ?: firstContainerId, + jobId = "job-123", seq = id ?: firstContainerIdInt, containerType = vmContainerType, status = status, diff --git a/src/backend/ci/core/process/biz-process/src/test/kotlin/com/tencent/devops/process/engine/control/CallBackControlTest.kt b/src/backend/ci/core/process/biz-process/src/test/kotlin/com/tencent/devops/process/engine/control/CallBackControlTest.kt index c0b6cf7813f..b64f6c1c90b 100644 --- a/src/backend/ci/core/process/biz-process/src/test/kotlin/com/tencent/devops/process/engine/control/CallBackControlTest.kt +++ b/src/backend/ci/core/process/biz-process/src/test/kotlin/com/tencent/devops/process/engine/control/CallBackControlTest.kt @@ -111,7 +111,9 @@ class CallBackControlTest : TestBase() { initBuildStartEnd(CallBackEvent.BUILD_START) val buildStartEvent = PipelineBuildStatusBroadCastEvent( source = "vm-build-claim($firstContainerId)", projectId = projectId, pipelineId = pipelineId, - userId = userId, buildId = buildId, actionType = ActionType.START + userId = userId, buildId = buildId, actionType = ActionType.START, stageId = null, + containerHashId = null, jobId = null, taskId = null, stepId = null, executeCount = null, + buildStatus = null ) // val startTime = System.currentTimeMillis() diff --git a/src/backend/ci/core/process/biz-process/src/test/kotlin/com/tencent/devops/process/util/MatrixYamlCheckUtilsTest.kt b/src/backend/ci/core/process/biz-process/src/test/kotlin/com/tencent/devops/process/util/MatrixYamlCheckUtilsTest.kt index 6130c3f1add..3c77ab5069d 100644 --- a/src/backend/ci/core/process/biz-process/src/test/kotlin/com/tencent/devops/process/util/MatrixYamlCheckUtilsTest.kt +++ b/src/backend/ci/core/process/biz-process/src/test/kotlin/com/tencent/devops/process/util/MatrixYamlCheckUtilsTest.kt @@ -116,9 +116,11 @@ internal class MatrixYamlCheckUtilsTest { @Test fun checkYaml5() { - val yamlstr = JsonUtil.toJson(mapOf( - "strategy" to "\${{fromJSONasd(asd)}}" - )) + val yamlstr = JsonUtil.toJson( + mapOf( + "strategy" to "\${{fromJSONasd(asd)}}" + ) + ) val result = try { MatrixContextUtils.schemaCheck(yamlstr) false @@ -131,9 +133,11 @@ internal class MatrixYamlCheckUtilsTest { @Test fun checkYaml6() { - val yamlstr = JsonUtil.toJson(mapOf( - "strategy" to "\${{fromJSON(asd)}}" - )) + val yamlstr = JsonUtil.toJson( + mapOf( + "strategy" to "\${{fromJSON(asd)}}" + ) + ) MatrixContextUtils.schemaCheck(yamlstr) } @@ -169,4 +173,17 @@ internal class MatrixYamlCheckUtilsTest { Assertions.assertTrue(result.exclude == null) Assertions.assertTrue(result.strategy == null) } + + @Test + fun checkYaml9() { + val yamlstr = MatrixPipelineInfo( + include = "\${{ fromJSON(FTP_DEPLOY_MODULE_LIST) }}", + exclude = "", + strategy = "\${{ fromJSON(FTP_DEPLOY_MODULE_NAMES) }}" + ) + /*测试并发*/ + List(1000) { it }.parallelStream().forEach { + MatrixYamlCheckUtils.checkYaml(yamlstr) + } + } } diff --git a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/api/op/OPProjectResource.kt b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/api/op/OPProjectResource.kt index f6e9fd295de..ad16ec6324a 100644 --- a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/api/op/OPProjectResource.kt +++ b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/api/op/OPProjectResource.kt @@ -136,6 +136,9 @@ interface OPProjectResource { @Parameter(description = "运营产品ID", required = true) @QueryParam(value = "product_id") productId: Int?, + @Parameter(description = "渠道", required = true) + @QueryParam(value = "channelCode") + channelCode: String?, @Context request: HttpServletRequest ): Result?> diff --git a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/api/open/OpenProjectResource.kt b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/api/open/OpenProjectResource.kt index 9abf65cd32a..55072b9b2e9 100644 --- a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/api/open/OpenProjectResource.kt +++ b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/api/open/OpenProjectResource.kt @@ -31,9 +31,9 @@ import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_BK_TOKEN import com.tencent.devops.project.pojo.OperationalProductVO import com.tencent.devops.project.pojo.ProjectVO import com.tencent.devops.project.pojo.Result -import io.swagger.v3.oas.annotations.tags.Tag import io.swagger.v3.oas.annotations.Operation import io.swagger.v3.oas.annotations.Parameter +import io.swagger.v3.oas.annotations.tags.Tag import javax.ws.rs.Consumes import javax.ws.rs.GET import javax.ws.rs.HeaderParam diff --git a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/api/service/ServiceUserResource.kt b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/api/service/ServiceUserResource.kt index b2cab4e36cb..483b19e070b 100644 --- a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/api/service/ServiceUserResource.kt +++ b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/api/service/ServiceUserResource.kt @@ -78,4 +78,13 @@ interface ServiceUserResource { @Parameter(description = "用户ID列表", required = true) userIds: List ): Result> + + @GET + @Path("/parentIds/{parentId}/usernames") + @Operation(summary = "根据父节点ID获取用户列表") + fun usernamesByParentId( + @Parameter(description = "父节点ID", required = true) + @PathParam("parentId") + parentId: Int + ): Result> } diff --git a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/constant/ProjectMessageCode.kt b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/constant/ProjectMessageCode.kt index 93dd58e4fcf..02624f431b6 100644 --- a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/constant/ProjectMessageCode.kt +++ b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/constant/ProjectMessageCode.kt @@ -50,7 +50,7 @@ object ProjectMessageCode { const val PROJECT_NAME_EXIST = "2119002" // 项目名或英文名重复 const val NAME_EMPTY = "2119003" // 名称不能为空 const val NAME_TOO_LONG = "2119004" // 项目名至多1-64个字符 - const val EN_NAME_INTERVAL_ERROR = "2119005" // 英文名长度在3-64个字符 + const val EN_NAME_INTERVAL_ERROR = "2119005" // 英文名长度在3-32个字符 const val EN_NAME_COMBINATION_ERROR = "2119006" // 英文名是字符+数字组成，并以小写字母开头 const val EN_NAME_EXIST = "2119007" // 英文名已经存在 const val PEM_CREATE_FAIL = "2119008" // 权限中心创建项目失败 diff --git a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectCreateUserInfo.kt b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectCreateUserInfo.kt index bf5c6202a14..3eadd66c675 100644 --- a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectCreateUserInfo.kt +++ b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectCreateUserInfo.kt @@ -33,7 +33,7 @@ import io.swagger.v3.oas.annotations.media.Schema @Schema data class ProjectCreateUserInfo( @get:Schema(title = "操作人") - val createUserId: String, + val createUserId: String?, @get:Schema(title = "待分配的角色名称") val roleName: String?, @get:Schema(title = "角色Id") diff --git a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectDeleteUserInfo.kt b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectDeleteUserInfo.kt index f116a86a512..66f06dfac97 100644 --- a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectDeleteUserInfo.kt +++ b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectDeleteUserInfo.kt @@ -33,7 +33,7 @@ import io.swagger.v3.oas.annotations.media.Schema @Schema data class ProjectDeleteUserInfo( @get:Schema(title = "操作人") - val operator: String, + val operator: String?, @get:Schema(title = "待分配的角色名称") val roleName: String?, @get:Schema(title = "角色Id") diff --git a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectDiffVO.kt b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectDiffVO.kt index e8f658ea271..4a61a381064 100644 --- a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectDiffVO.kt +++ b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectDiffVO.kt @@ -116,5 +116,9 @@ data class ProjectDiffVO( @get:Schema(title = "运营产品ID") val productId: Int? = null, @get:Schema(title = "审批中运营产品ID") - val afterProductId: Int? = null + val afterProductId: Int? = null, + @get:Schema(title = "运营产品名称") + val productName: String? = null, + @get:Schema(title = "审批中运营产品名称") + val afterProductName: String? = null ) diff --git a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectProperties.kt b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectProperties.kt index 85b4eb703d6..2d33acb0ac4 100644 --- a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectProperties.kt +++ b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectProperties.kt @@ -47,5 +47,7 @@ data class ProjectProperties( @get:Schema(title = "数据标签，创建项目时会为该项目分配指定标签的db") val dataTag: String? = null, @get:Schema(title = "当项目不活跃时，是否禁用") - var disableWhenInactive: Boolean? = null + var disableWhenInactive: Boolean? = null, + @get:Schema(title = "该项目是否开启流水线可观测数据", required = false) + val buildMetrics: Boolean? = null ) diff --git a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectVO.kt b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectVO.kt index c99b848a151..82362f1fe4a 100644 --- a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectVO.kt +++ b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/ProjectVO.kt @@ -154,6 +154,8 @@ data class ProjectVO( val channelCode: String? = null, @get:Schema(title = "运营产品ID") val productId: Int? = null, + @get:Schema(title = "运营产品名称") + val productName: String? = null, @get:Schema(title = "是否可以查看") val canView: Boolean? = null, @get:Schema(title = "安装模板权限") diff --git a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/service/ServiceVO.kt b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/service/ServiceVO.kt index 37c7c728f3a..9b02f9e34d7 100644 --- a/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/service/ServiceVO.kt +++ b/src/backend/ci/core/project/api-project/src/main/kotlin/com/tencent/devops/project/pojo/service/ServiceVO.kt @@ -87,5 +87,7 @@ data class ServiceVO( @get:Schema(title = "new_window_url") val newWindowUrl: String? = null, @get:Schema(title = "集群类型") - val clusterType: String = "" + val clusterType: String = "", + @get:Schema(title = "文档链接") + val docUrl: String = "" ) diff --git a/src/backend/ci/core/project/biz-project-sample/src/main/kotlin/com/tencent/devops/project/service/SimpleProjectServiceImpl.kt b/src/backend/ci/core/project/biz-project-sample/src/main/kotlin/com/tencent/devops/project/service/SimpleProjectServiceImpl.kt index 6c545389843..538a9a38ae7 100644 --- a/src/backend/ci/core/project/biz-project-sample/src/main/kotlin/com/tencent/devops/project/service/SimpleProjectServiceImpl.kt +++ b/src/backend/ci/core/project/biz-project-sample/src/main/kotlin/com/tencent/devops/project/service/SimpleProjectServiceImpl.kt @@ -213,16 +213,23 @@ class SimpleProjectServiceImpl @Autowired constructor( return listOf( OperationalProductVO( productId = -1, - productName = "其他" + productName = "other" ) ) } + override fun getProductByProductId(productId: Int): OperationalProductVO? { + return OperationalProductVO( + productId = -1, + productName = "other" + ) + } + override fun getOperationalProductsByBgName(bgName: String): List { return listOf( OperationalProductVO( productId = -1, - productName = "其他" + productName = "other" ) ) } diff --git a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/dao/ProjectDao.kt b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/dao/ProjectDao.kt index 68f29f27710..58cadb34420 100644 --- a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/dao/ProjectDao.kt +++ b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/dao/ProjectDao.kt @@ -523,7 +523,8 @@ class ProjectDao { routerTag: String?, otherRouterTagMaps: Map?, remoteDevFlag: Boolean?, - productId: Int? + productId: Int?, + channelCode: String? ): MutableList { val conditions = mutableListOf() if (!projectName.isNullOrBlank()) { @@ -551,6 +552,7 @@ class ProjectDao { conditions.add(CHANNEL.eq(ProjectChannelCode.BS.name)) conditions.add(JooqUtils.jsonExtractAny(PROPERTIES, "\$.remotedev").isTrue) } + channelCode?.let { conditions.add(CHANNEL.eq(channelCode)) } return conditions } @@ -568,7 +570,8 @@ class ProjectDao { routerTag: String? = null, otherRouterTagMaps: Map? = null, remoteDevFlag: Boolean? = null, - productId: Int? = null + productId: Int? = null, + channelCode: String? = null ): Result { with(TProject.T_PROJECT) { val conditions = generateQueryProjectCondition( @@ -582,7 +585,8 @@ class ProjectDao { routerTag = routerTag, otherRouterTagMaps = otherRouterTagMaps, remoteDevFlag = remoteDevFlag, - productId = productId + productId = productId, + channelCode = channelCode ) return dslContext.selectFrom(this).where(conditions).orderBy(CREATED_AT.desc()).limit(offset, limit).fetch() } @@ -752,7 +756,8 @@ class ProjectDao { routerTag: String? = null, otherRouterTagMaps: Map? = null, remoteDevFlag: Boolean? = null, - productId: Int? = null + productId: Int? = null, + channelCode: String? = null ): Int { with(TProject.T_PROJECT) { val conditions = generateQueryProjectCondition( @@ -766,7 +771,8 @@ class ProjectDao { routerTag = routerTag, otherRouterTagMaps = otherRouterTagMaps, remoteDevFlag = remoteDevFlag, - productId = productId + productId = productId, + channelCode = channelCode ) return dslContext.selectCount().from(this).where(conditions).fetchOne(0, Int::class.java)!! } diff --git a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/dao/UserDao.kt b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/dao/UserDao.kt index c22f097e258..354e4bb5d44 100644 --- a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/dao/UserDao.kt +++ b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/dao/UserDao.kt @@ -30,10 +30,10 @@ package com.tencent.devops.project.dao import com.tencent.devops.model.project.tables.TUser import com.tencent.devops.model.project.tables.records.TUserRecord import com.tencent.devops.project.pojo.user.UserDeptDetail -import java.time.LocalDateTime import org.jooq.DSLContext import org.jooq.Result import org.springframework.stereotype.Repository +import java.time.LocalDateTime @Suppress("ALL") @Repository @@ -138,4 +138,16 @@ class UserDao { businessLineName = userRecord.businessLineName ) } + + fun usernamesByParentId(dslContext: DSLContext, parentId: Int): List { + with(TUser.T_USER) { + return dslContext.select(USER_ID) + .from(this) + .where(BG_ID.eq(parentId)) + .or(DEPT_ID.eq(parentId)) + .or(CENTER_ID.eq(parentId)) + .or(GROYP_ID.eq(parentId)) + .fetch(0, String::class.java) + } + } } diff --git a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/resources/OPProjectResourceImpl.kt b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/resources/OPProjectResourceImpl.kt index d18663184f4..2eb765e0931 100644 --- a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/resources/OPProjectResourceImpl.kt +++ b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/resources/OPProjectResourceImpl.kt @@ -104,6 +104,7 @@ class OPProjectResourceImpl @Autowired constructor( repoGrayFlag: Boolean, remoteDevFlag: Boolean, productId: Int?, + channelCode: String?, request: HttpServletRequest ): Result?> { return Result( @@ -121,7 +122,8 @@ class OPProjectResourceImpl @Autowired constructor( codeCCGrayFlag = codeCCGrayFlag, repoGrayFlag = repoGrayFlag, remoteDevFlag = remoteDevFlag, - productId = productId + productId = productId, + channelCode = channelCode ) ) } diff --git a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/resources/OpenProjectResourceImpl.kt b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/resources/OpenProjectResourceImpl.kt index c103bf8bb56..ffe84033722 100644 --- a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/resources/OpenProjectResourceImpl.kt +++ b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/resources/OpenProjectResourceImpl.kt @@ -31,6 +31,8 @@ import com.tencent.devops.common.api.exception.OperationException import com.tencent.devops.common.api.exception.TokenForbiddenException import com.tencent.devops.common.client.ClientTokenService import com.tencent.devops.common.web.RestResource +import com.tencent.devops.common.web.annotation.BkApiPermission +import com.tencent.devops.common.web.constant.BkApiHandleType import com.tencent.devops.project.api.open.OpenProjectResource import com.tencent.devops.project.pojo.OperationalProductVO import com.tencent.devops.project.pojo.ProjectVO @@ -43,6 +45,7 @@ class OpenProjectResourceImpl constructor( private val projectService: ProjectService, private val clientTokenService: ClientTokenService ) : OpenProjectResource { + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun get( token: String, projectId: String @@ -55,6 +58,7 @@ class OpenProjectResourceImpl constructor( ) } + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun listByProjectCodes( token: String, projectCodes: Set @@ -68,6 +72,7 @@ class OpenProjectResourceImpl constructor( ) } + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun getOperationalProducts(token: String): Result> { check(token) return Result(projectService.getOperationalProducts()) diff --git a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/resources/ServiceUserResourceImpl.kt b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/resources/ServiceUserResourceImpl.kt index df05be3e098..4433481e839 100644 --- a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/resources/ServiceUserResourceImpl.kt +++ b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/resources/ServiceUserResourceImpl.kt @@ -55,4 +55,8 @@ class ServiceUserResourceImpl @Autowired constructor( override fun listDetailFromCache(userIds: List): Result> { return Result(userCacheService.listDetailFromCache(userIds)) } + + override fun usernamesByParentId(parentId: Int): Result> { + return Result(userCacheService.usernamesByParentIds(parentId)) + } } diff --git a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/resources/UserProjectResourceImpl.kt b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/resources/UserProjectResourceImpl.kt index d5c924efe29..827713fc742 100644 --- a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/resources/UserProjectResourceImpl.kt +++ b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/resources/UserProjectResourceImpl.kt @@ -120,8 +120,7 @@ class UserProjectResourceImpl @Autowired constructor( userId = userId, englishName = projectId, accessToken = accessToken - ) - ?: throw OperationException("project $projectId not found") + ) ?: throw OperationException("project $projectId not found") ) } diff --git a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/OpProjectService.kt b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/OpProjectService.kt index 4be68de3dcd..3295abaffd8 100644 --- a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/OpProjectService.kt +++ b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/OpProjectService.kt @@ -56,6 +56,7 @@ interface OpProjectService { codeCCGrayFlag: Boolean, repoGrayFlag: Boolean, remoteDevFlag: Boolean, - productId: Int? + productId: Int?, + channelCode: String? ): Map? } diff --git a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/ProjectService.kt b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/ProjectService.kt index fc0acbb3e6b..967e9c1169f 100644 --- a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/ProjectService.kt +++ b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/ProjectService.kt @@ -253,6 +253,8 @@ interface ProjectService { fun getOperationalProducts(): List + fun getProductByProductId(productId: Int): OperationalProductVO? + fun getOperationalProductsByBgName(bgName: String): List fun updateProjectProductId( diff --git a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/ProjectTagService.kt b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/ProjectTagService.kt index baae3685482..b95488fbcd9 100644 --- a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/ProjectTagService.kt +++ b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/ProjectTagService.kt @@ -80,6 +80,12 @@ class ProjectTagService @Autowired constructor( @Value("\${tag.gray:#{null}}") private val grayTag: String? = null + @Value("\${tag.codecc.gray:#{null}}") + private val codeccGrayTag: String? = null + + @Value("\${tag.codecc.prod:#{null}}") + private val codeccProdTag: String? = null + @Value("\${system.inContainer:#{null}}") private val inContainerTags: String? = null @@ -90,8 +96,22 @@ class ProjectTagService @Autowired constructor( fun setGrayExt(projectCodeList: List, operateFlag: Int, system: SystemEnums): Boolean { val routerTag = when (operateFlag) { - grayLabel -> grayTag - prodLabel -> prodTag + grayLabel -> { + if (system == SystemEnums.CODECC) { + codeccGrayTag + } else { + grayTag + } + } + + prodLabel -> { + if (system == SystemEnums.CODECC) { + codeccProdTag + } else { + grayTag + } + } + else -> null } diff --git a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/UserCacheService.kt b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/UserCacheService.kt index ca892874964..12714f03ff5 100644 --- a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/UserCacheService.kt +++ b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/UserCacheService.kt @@ -44,16 +44,21 @@ class UserCacheService @Autowired constructor( */ fun getDetailFromCache(userId: String): UserDeptDetail { val userRecord = userDao.get(dslContext, userId) - return getUserDeptDetail(userRecord) + return getUserDeptDetail(userRecord, userId) } fun listDetailFromCache(userIds: List): List { return userDao.list(dslContext, userIds).map { getUserDeptDetail(it) } } - fun getUserDeptDetail(userRecord: TUserRecord?): UserDeptDetail { + fun usernamesByParentIds(parentId: Int): List { + return userDao.usernamesByParentId(dslContext, parentId) + } + + fun getUserDeptDetail(userRecord: TUserRecord?, userId: String? = null): UserDeptDetail { return if (userRecord == null) { UserDeptDetail( + userId = userId, bgName = "", bgId = "0", centerName = "", diff --git a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/impl/AbsOpProjectServiceImpl.kt b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/impl/AbsOpProjectServiceImpl.kt index 8f5a627b7cc..8ce21a4ff33 100644 --- a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/impl/AbsOpProjectServiceImpl.kt +++ b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/impl/AbsOpProjectServiceImpl.kt @@ -67,6 +67,8 @@ abstract class AbsOpProjectServiceImpl @Autowired constructor( @Value("\${tag.gray:#{null}}") private val grayTag: String? = null + @Value("\${tag.codecc.gray:#{null}}") + private val codeccGrayTag: String? = null override fun updateProjectFromOp( userId: String, @@ -171,15 +173,16 @@ abstract class AbsOpProjectServiceImpl @Autowired constructor( codeCCGrayFlag: Boolean, repoGrayFlag: Boolean, remoteDevFlag: Boolean, - productId: Int? + productId: Int?, + channelCode: String? ): Map? { val dataObj = mutableMapOf() val routerTag = if (grayFlag) grayTag else null val otherRouterTagMaps = mutableMapOf() - if (codeCCGrayFlag && grayTag != null) { - otherRouterTagMaps[SystemEnums.CODECC.name] = grayTag + if (codeCCGrayFlag && codeccGrayTag != null) { + otherRouterTagMaps[SystemEnums.CODECC.name] = codeccGrayTag } if (repoGrayFlag && grayTag != null) { otherRouterTagMaps[SystemEnums.REPO.name] = grayTag @@ -204,7 +207,8 @@ abstract class AbsOpProjectServiceImpl @Autowired constructor( routerTag = routerTag, otherRouterTagMaps = otherRouterTagMaps, remoteDevFlag = remoteDevFlag, - productId = productId + productId = productId, + channelCode = channelCode ) val totalCount = projectDao.getProjectCount( dslContext = dslContext, @@ -218,7 +222,8 @@ abstract class AbsOpProjectServiceImpl @Autowired constructor( routerTag = routerTag, otherRouterTagMaps = otherRouterTagMaps, remoteDevFlag = remoteDevFlag, - productId = productId + productId = productId, + channelCode = channelCode ) val dataList = mutableListOf() @@ -273,7 +278,7 @@ abstract class AbsOpProjectServiceImpl @Autowired constructor( kind = kind, enabled = enabled ?: true, grayFlag = routerTag == grayTag, - codeCCGrayFlag = otherRouterTagMap[SystemEnums.CODECC.name] == grayTag, + codeCCGrayFlag = otherRouterTagMap[SystemEnums.CODECC.name] == codeccGrayTag, repoGrayFlag = otherRouterTagMap[SystemEnums.REPO.name] == grayTag, hybridCCAppId = hybridCcAppId, enableExternal = enableExternal, diff --git a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/impl/AbsProjectServiceImpl.kt b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/impl/AbsProjectServiceImpl.kt index c6d608aa521..c927c1ab717 100644 --- a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/impl/AbsProjectServiceImpl.kt +++ b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/impl/AbsProjectServiceImpl.kt @@ -27,6 +27,7 @@ package com.tencent.devops.project.service.impl +import com.fasterxml.jackson.core.type.TypeReference import com.fasterxml.jackson.databind.ObjectMapper import com.tencent.bk.audit.annotations.ActionAuditRecord import com.tencent.bk.audit.annotations.AuditInstanceRecord @@ -80,6 +81,7 @@ import com.tencent.devops.project.jmx.api.ProjectJmxApi import com.tencent.devops.project.jmx.api.ProjectJmxApi.Companion.PROJECT_LIST import com.tencent.devops.project.pojo.AuthProjectCreateInfo import com.tencent.devops.project.pojo.ProjectBaseInfo +import com.tencent.devops.project.pojo.ProjectByConditionDTO import com.tencent.devops.project.pojo.ProjectCollation import com.tencent.devops.project.pojo.ProjectCreateExtInfo import com.tencent.devops.project.pojo.ProjectCreateInfo @@ -93,7 +95,6 @@ import com.tencent.devops.project.pojo.ProjectUpdateCreatorDTO import com.tencent.devops.project.pojo.ProjectUpdateHistoryInfo import com.tencent.devops.project.pojo.ProjectUpdateInfo import com.tencent.devops.project.pojo.ProjectVO -import com.tencent.devops.project.pojo.ProjectByConditionDTO import com.tencent.devops.project.pojo.ResourceUpdateInfo import com.tencent.devops.project.pojo.Result import com.tencent.devops.project.pojo.enums.ProjectApproveStatus @@ -447,7 +448,10 @@ abstract class AbsProjectServiceImpl @Autowired constructor( } } val tipsStatus = getAndUpdateTipsStatus(userId = userId, projectId = englishName) - return projectInfo.copy(tipsStatus = tipsStatus) + return projectInfo.copy( + tipsStatus = tipsStatus, + productName = projectInfo.productId?.let { getProductByProductId(it)?.productName } + ) } protected fun getAndUpdateTipsStatus(userId: String, projectId: String): Int { @@ -473,10 +477,16 @@ abstract class AbsProjectServiceImpl @Autowired constructor( val record = projectDao.getByEnglishName(dslContext, englishName) ?: return null val projectApprovalInfo = projectApprovalService.get(englishName) val rightProjectOrganization = fixProjectOrganization(tProjectRecord = record) + val beforeProductName = if (record.productId != null) { + getProductByProductId(record.productId) + } else { + null + } return ProjectUtils.packagingBean( tProjectRecord = record, projectApprovalInfo = projectApprovalInfo, - projectOrganizationInfo = rightProjectOrganization + projectOrganizationInfo = rightProjectOrganization, + beforeProductName = beforeProductName?.productName ) } @@ -542,7 +552,7 @@ abstract class AbsProjectServiceImpl @Autowired constructor( val (finalNeedApproval, newApprovalStatus) = getUpdateApprovalStatus( needApproval = needApproval, projectInfo = projectInfo, - subjectScopesStr = subjectScopesStr, + afterSubjectScopes = subjectScopes, projectUpdateInfo = projectUpdateInfo ) val projectId = projectInfo.projectId @@ -558,8 +568,8 @@ abstract class AbsProjectServiceImpl @Autowired constructor( originalProjectName = projectInfo.projectName, modifiedProjectName = projectUpdateInfo.projectName, finalNeedApproval = finalNeedApproval, - beforeSubjectScopesStr = projectInfo.subjectScopes, - afterSubjectScopesStr = subjectScopesStr + beforeSubjectScopes = JsonUtil.to(projectInfo.subjectScopes, object : TypeReference>() {}), + afterSubjectScopes = subjectScopes )) { modifyProjectAuthResource(resourceUpdateInfo) } @@ -692,29 +702,35 @@ abstract class AbsProjectServiceImpl @Autowired constructor( originalProjectName: String, modifiedProjectName: String, finalNeedApproval: Boolean, - beforeSubjectScopesStr: String, - afterSubjectScopesStr: String + beforeSubjectScopes: List, + afterSubjectScopes: List ): Boolean { + val isSubjectScopesChange = isSubjectScopesChange( + beforeSubjectScopes = beforeSubjectScopes, + afterSubjectScopes = afterSubjectScopes + ) return originalProjectName != modifiedProjectName || finalNeedApproval || - beforeSubjectScopesStr != afterSubjectScopesStr + isSubjectScopesChange } private fun getUpdateApprovalStatus( needApproval: Boolean?, projectInfo: TProjectRecord, - subjectScopesStr: String, + afterSubjectScopes: List, projectUpdateInfo: ProjectUpdateInfo ): Pair { val authNeedApproval = projectPermissionService.needApproval(needApproval) val approveStatus = ProjectApproveStatus.parse(projectInfo.approvalStatus) // 判断是否需要审批 return if (approveStatus.isSuccess()) { + val isSubjectScopesChange = isSubjectScopesChange( + beforeSubjectScopes = JsonUtil.to(projectInfo.subjectScopes, object : TypeReference>() {}), + afterSubjectScopes = afterSubjectScopes + ) // 当项目创建成功,则只有最大授权范围和项目性质修改才审批 val finalNeedApproval = authNeedApproval && - (projectInfo.subjectScopes != subjectScopesStr || - projectInfo.authSecrecy != projectUpdateInfo.authSecrecy || - projectInfo.productId != projectUpdateInfo.productId - ) + (isSubjectScopesChange || projectInfo.authSecrecy != projectUpdateInfo.authSecrecy || + projectInfo.productId != projectUpdateInfo.productId) val approvalStatus = if (finalNeedApproval) { ProjectApproveStatus.UPDATE_PENDING.status } else { @@ -727,6 +743,15 @@ abstract class AbsProjectServiceImpl @Autowired constructor( } } + private fun isSubjectScopesChange( + beforeSubjectScopes: List, + afterSubjectScopes: List + ): Boolean { + val beforeIds = beforeSubjectScopes.map { it.id }.toSet() + val afterIds = afterSubjectScopes.map { it.id }.toSet() + return beforeIds != afterIds + } + private fun updateApprovalInfo( userId: String, projectId: String, diff --git a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/impl/AbsUserProjectServiceServiceImpl.kt b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/impl/AbsUserProjectServiceServiceImpl.kt index 1cdd2c9a71c..c8fecff0fc8 100644 --- a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/impl/AbsUserProjectServiceServiceImpl.kt +++ b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/service/impl/AbsUserProjectServiceServiceImpl.kt @@ -86,7 +86,8 @@ abstract class AbsUserProjectServiceServiceImpl @Autowired constructor( weigHt = tServiceRecord.weight ?: 0, logoUrl = tServiceRecord.logoUrl, webSocket = tServiceRecord.webSocket, - clusterType = tServiceRecord.clusterType + clusterType = tServiceRecord.clusterType, + docUrl = tServiceRecord.docUrl ?: "" ) ) } else { @@ -251,7 +252,8 @@ abstract class AbsUserProjectServiceServiceImpl @Autowired constructor( webSocket = it.webSocket, newWindow = it.newWindow, newWindowUrl = it.newWindowurl, - clusterType = it.clusterType + clusterType = it.clusterType, + docUrl = it.docUrl ?: "" ) ) } diff --git a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/util/ProjectUtils.kt b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/util/ProjectUtils.kt index e13c8dcfbc3..4d21fac15f5 100644 --- a/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/util/ProjectUtils.kt +++ b/src/backend/ci/core/project/biz-project/src/main/kotlin/com/tencent/devops/project/util/ProjectUtils.kt @@ -128,7 +128,8 @@ object ProjectUtils { fun packagingBean( tProjectRecord: TProjectRecord, projectApprovalInfo: ProjectApprovalInfo?, - projectOrganizationInfo: ProjectOrganizationInfo? = null + projectOrganizationInfo: ProjectOrganizationInfo? = null, + beforeProductName: String? = null ): ProjectDiffVO { val isUseFixedOrganization = projectOrganizationInfo != null val subjectScopes = tProjectRecord.subjectScopes?.let { @@ -187,7 +188,9 @@ object ProjectUtils { projectType = projectType, afterProjectType = projectApprovalInfo?.projectType, productId = productId, - afterProductId = projectApprovalInfo?.productId + afterProductId = projectApprovalInfo?.productId, + productName = beforeProductName, + afterProductName = projectApprovalInfo?.productName ) } } diff --git a/src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/service/v2/QualityRuleCheckService.kt b/src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/service/v2/QualityRuleCheckService.kt index 76e86d3328e..1b817a87878 100644 --- a/src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/service/v2/QualityRuleCheckService.kt +++ b/src/backend/ci/core/quality/biz-quality/src/main/kotlin/com/tencent/devops/quality/service/v2/QualityRuleCheckService.kt @@ -46,6 +46,7 @@ import com.tencent.devops.notify.PIPELINE_QUALITY_END_NOTIFY_TEMPLATE_V2 import com.tencent.devops.notify.api.service.ServiceNotifyMessageTemplateResource import com.tencent.devops.notify.pojo.SendNotifyMessageTemplateRequest import com.tencent.devops.plugin.codecc.CodeccUtils +import com.tencent.devops.process.api.service.ServiceVarResource import com.tencent.devops.process.utils.PIPELINE_NAME import com.tencent.devops.process.utils.PIPELINE_START_USER_ID import com.tencent.devops.process.utils.PIPELINE_START_USER_NAME @@ -257,7 +258,6 @@ class QualityRuleCheckService @Autowired constructor( private fun doCheckRules(buildCheckParams: BuildCheckParams, ruleList: List): RuleCheckResult { with(buildCheckParams) { - logger.info("QUALITY|doCheckRules buildCheckParams is|$buildCheckParams") val filterRuleList = ruleList.filter { rule -> logger.info("validate whether to check rule(${rule.name}) with gatewayId(${rule.gatewayId})") if (buildCheckParams.taskId.isNotBlank() && rule.controlPoint.name != buildCheckParams.taskId) { @@ -325,14 +325,14 @@ class QualityRuleCheckService @Autowired constructor( "buildId" to buildId, CodeccUtils.BK_CI_CODECC_TASK_ID to (runtimeVariable?.get(CodeccUtils.BK_CI_CODECC_TASK_ID) ?: "") - ) + ).toMutableMap() // 指标详情链接支持占位符 interceptRecordList.forEach { record -> record.logPrompt = runtimeVariable?.let { EnvUtils.parseEnv(record.logPrompt, it) } ?: record.logPrompt } - resultList.add(getRuleCheckSingleResult(rule.name, interceptRecordList, params)) + resultList.add(getRuleCheckSingleResult(rule.name, interceptRecordList, params, result.third)) ruleInterceptList.add(Triple(rule, interceptResult, interceptRecordList)) val status = if (interceptResult) { @@ -487,7 +487,7 @@ class QualityRuleCheckService @Autowired constructor( indicators: List, metadataList: List, ruleTaskSteps: List? - ): Pair> { + ): Triple, Set> { var allCheckResult = true val interceptList = mutableListOf() var ruleTaskStepsCopy = ruleTaskSteps?.toMutableList() @@ -502,9 +502,9 @@ class QualityRuleCheckService @Autowired constructor( logger.info("QUALITY|metadataList is: $metadataList, indicators is:$indicators") - val (indicatorsCopy, metadataListCopy) = handleWithMultiIndicator(indicators, metadataList) + val (indicatorsCopy, metadataListCopy, taskAtomMap) = handleWithMultiIndicator(indicators, metadataList) - logger.info("QUALITY|indicatorsCopy is:$indicatorsCopy") + logger.info("QUALITY|indicatorsCopy is:$indicatorsCopy, task atom map is: $taskAtomMap") // 遍历每个指标 indicatorsCopy.forEach { indicator -> val thresholdType = indicator.thresholdType @@ -655,12 +655,13 @@ class QualityRuleCheckService @Autowired constructor( actualValue = result, pass = checkResult, detail = elementDetail, - logPrompt = logPrompt + logPrompt = logPrompt, + controlPointElementId = taskAtomMap[indicator.taskName] ) ) } } - return Pair(allCheckResult, interceptList) + return Triple(allCheckResult, interceptList, taskAtomMap.values.toSet()) } /** @@ -669,8 +670,20 @@ class QualityRuleCheckService @Autowired constructor( private fun getRuleCheckSingleResult( ruleName: String, interceptRecordList: List, - params: Map + params: MutableMap, + elementIdSet: Set ): RuleCheckSingleResult { + // 为防止相同插件的并发问题，在生成问题链接时从var表查询taskId + val variable = if (elementIdSet.isNotEmpty()) { + client.get(ServiceVarResource::class).getBuildVars( + projectId = params["projectId"]!!, + pipelineId = params["pipelineId"]!!, + buildId = params["buildId"]!!, + keys = elementIdSet.map { CodeccUtils.BK_CI_CODECC_ATOM_ID_TO_TASK_ID + "_" + it }.toSet() + ).data + } else null + params.putAll(variable ?: mapOf()) + logger.info("rule check result param is: $params") val messageList = interceptRecordList.map { val thresholdOperationName = ThresholdOperationUtil.getOperationName(it.operation) @@ -707,8 +720,9 @@ class QualityRuleCheckService @Autowired constructor( val projectId = params["projectId"] ?: "" val pipelineId = params["pipelineId"] ?: "" val buildId = params["buildId"] ?: "" - val taskId = params[CodeccUtils.BK_CI_CODECC_TASK_ID] ?: "" - if (taskId.isBlank()) { + val codeccAtomKey = "${CodeccUtils.BK_CI_CODECC_ATOM_ID_TO_TASK_ID}_${record.controlPointElementId}" + val taskId = params[codeccAtomKey] ?: params[CodeccUtils.BK_CI_CODECC_TASK_ID] + if (taskId.isNullOrBlank()) { logger.warn("taskId is null or blank for project($projectId) pipeline($pipelineId)") return "" } @@ -954,9 +968,10 @@ class QualityRuleCheckService @Autowired constructor( private fun handleWithMultiIndicator( indicators: List, metadataList: List - ): Pair, List> { + ): Triple, List, Map> { val indicatorsCopy = indicators.toMutableList() val metadataListCopy = metadataList.map { it.clone() } + val taskAtomMap = mutableMapOf() // // CodeCC插件一个指标的元数据对应多条，先对多个CodeCC插件提前做标识 val codeccMetaList = metadataListCopy.filter { @@ -964,7 +979,10 @@ class QualityRuleCheckService @Autowired constructor( }.groupBy { it.taskId } if (codeccMetaList.size > 1) { codeccMetaList.values.forEachIndexed { index, codeccMeta -> - codeccMeta.map { it.taskName = "${it.taskName}+$index" } + codeccMeta.map { + it.taskName = "${it.taskName}+$index" + taskAtomMap.put(it.taskName, it.taskId) + } } } @@ -1012,7 +1030,7 @@ class QualityRuleCheckService @Autowired constructor( } } } - return Pair(indicatorsCopy, metadataListCopy) + return Triple(indicatorsCopy, metadataListCopy, taskAtomMap) } private fun handleScriptAndThirdPlugin( diff --git a/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/api/ServiceRepositoryPacResource.kt b/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/api/ServiceRepositoryPacResource.kt index a64c78b6bcb..9befd8ea37b 100644 --- a/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/api/ServiceRepositoryPacResource.kt +++ b/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/api/ServiceRepositoryPacResource.kt @@ -28,6 +28,8 @@ package com.tencent.devops.repository.api +import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID +import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID_DEFAULT_VALUE import com.tencent.devops.common.api.enums.ScmType import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.repository.pojo.Repository @@ -37,10 +39,12 @@ import io.swagger.v3.oas.annotations.Parameter import javax.ws.rs.Consumes import javax.ws.rs.GET import javax.ws.rs.POST +import javax.ws.rs.PUT import javax.ws.rs.Path import javax.ws.rs.PathParam import javax.ws.rs.Produces import javax.ws.rs.QueryParam +import javax.ws.rs.HeaderParam import javax.ws.rs.core.MediaType @Tag(name = "SERVICE_PAC_REPOSITORY", description = "服务-PAC-代码库") @@ -74,4 +78,33 @@ interface ServiceRepositoryPacResource { @QueryParam("scmType") scmType: ScmType ): Result + @Operation(summary = "开启pac") + @PUT + @Path("/{projectId}/{repositoryHashId}/enable") + fun enablePac( + @Parameter(description = "用户ID", required = true, example = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @Parameter(description = "项目ID", required = true) + @PathParam("projectId") + projectId: String, + @Parameter(description = "代码库哈希ID", required = true) + @PathParam("repositoryHashId") + repositoryHashId: String + ): Result + + @Operation(summary = "关闭pac") + @PUT + @Path("/{projectId}/{repositoryHashId}/disable") + fun disablePac( + @Parameter(description = "用户ID", required = true, example = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @Parameter(description = "项目ID", required = true) + @PathParam("projectId") + projectId: String, + @Parameter(description = "代码库哈希ID", required = true) + @PathParam("repositoryHashId") + repositoryHashId: String + ): Result } diff --git a/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/api/ServiceRepositoryResource.kt b/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/api/ServiceRepositoryResource.kt index d45bfd7cce1..c66c7a262a8 100644 --- a/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/api/ServiceRepositoryResource.kt +++ b/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/api/ServiceRepositoryResource.kt @@ -255,4 +255,19 @@ interface ServiceRepositoryResource { @Parameter(description = "代码库哈希ID列表", required = true) repositoryHashIdList: List ): Result> + + @Operation(summary = "更新组件代码库关联项目信息") + @POST + @Path("/store/project/update") + fun updateStoreRepoProject( + @Parameter(description = "代码库负责人") + @QueryParam("userId") + userId: String, + @Parameter(description = "项目ID") + @QueryParam("projectId") + projectId: String, + @Parameter(description = "代码库ID") + @QueryParam("repositoryId") + repositoryId: Long + ): Result } diff --git a/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/api/UserRepositoryConfigResource.kt b/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/api/UserRepositoryConfigResource.kt new file mode 100644 index 00000000000..18d2b7cabee --- /dev/null +++ b/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/api/UserRepositoryConfigResource.kt @@ -0,0 +1,50 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.repository.api + +import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.repository.pojo.RepositoryConfig +import io.swagger.v3.oas.annotations.Operation +import io.swagger.v3.oas.annotations.tags.Tag +import javax.ws.rs.Consumes +import javax.ws.rs.GET +import javax.ws.rs.Path +import javax.ws.rs.Produces +import javax.ws.rs.core.MediaType + +@Tag(name = "USER_REPOSITORY_MANAGER", description = "用户-代码库配置") +@Path("/user/repositories/config") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +interface UserRepositoryConfigResource { + + @Operation(summary = "获取代码库配置列表") + @GET + @Path("/") + fun list(): Result> +} diff --git a/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/api/scm/ServiceScmOauthResource.kt b/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/api/scm/ServiceScmOauthResource.kt index 1fa8491f61d..266e7f8406c 100644 --- a/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/api/scm/ServiceScmOauthResource.kt +++ b/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/api/scm/ServiceScmOauthResource.kt @@ -122,7 +122,13 @@ interface ServiceScmOauthResource { userName: String?, @Parameter(description = "搜索条件", required = false) @QueryParam("search") - search: String? = null + search: String? = null, + @Parameter(description = "page", required = true) + @QueryParam("page") + page: Int = 1, + @Parameter(description = "pageSize", required = true) + @QueryParam("pageSize") + pageSize: Int = 20 ): Result> @Operation(summary = "List all the branches of repo") diff --git a/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/api/scm/ServiceTGitResource.kt b/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/api/scm/ServiceTGitResource.kt index 3e77f401b21..d727f275852 100644 --- a/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/api/scm/ServiceTGitResource.kt +++ b/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/api/scm/ServiceTGitResource.kt @@ -36,6 +36,7 @@ import com.tencent.devops.repository.pojo.git.GitCodeProjectInfo import com.tencent.devops.repository.pojo.git.GitUserInfo import com.tencent.devops.scm.code.git.api.GitBranch import com.tencent.devops.scm.enums.GitAccessLevelEnum +import com.tencent.devops.scm.pojo.ChangeFileInfo import com.tencent.devops.scm.pojo.GitFileInfo import io.swagger.v3.oas.annotations.tags.Tag import io.swagger.v3.oas.annotations.Operation @@ -186,4 +187,37 @@ interface ServiceTGitResource { @QueryParam("tokenType") tokenType: TokenTypeEnum = TokenTypeEnum.OAUTH ): Result + + @Operation(summary = "获取两次提交的差异文件列表") + @GET + @Path("/getChangeFileList") + fun getChangeFileList( + @Parameter(description = "token") + @QueryParam("token") + token: String, + @Parameter(description = "token类型 0：oauth 1:privateKey", required = true) + @QueryParam("tokenType") + tokenType: TokenTypeEnum, + @Parameter(description = "gitProjectId") + @QueryParam("gitProjectId") + gitProjectId: String, + @Parameter(description = "旧commit") + @QueryParam("from") + from: String, + @Parameter(description = "新commit") + @QueryParam("to") + to: String, + @Parameter(description = "true：两个点比较差异，false：三个点比较差异。默认是 false") + @QueryParam("straight") + straight: Boolean? = false, + @Parameter(description = "页码") + @QueryParam("page") + page: Int, + @Parameter(description = "每页大小") + @QueryParam("pageSize") + pageSize: Int, + @Parameter(description = "代码库url") + @QueryParam("url") + url: String + ): Result> } diff --git a/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/pojo/RepositoryConfig.kt b/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/pojo/RepositoryConfig.kt new file mode 100644 index 00000000000..7598617a5cc --- /dev/null +++ b/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/pojo/RepositoryConfig.kt @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.repository.pojo + +import com.tencent.devops.common.api.enums.ScmType +import com.tencent.devops.repository.pojo.enums.RepositoryConfigStatusEnum +import io.swagger.v3.oas.annotations.media.Schema + +@Schema(title = "代码库配置") +data class RepositoryConfig( + @get:Schema(title = "代码库类型", required = false) + val scmType: ScmType, + @get:Schema(title = "代码库名称", required = false) + val name: String, + @get:Schema(title = "代码库状态", required = false) + val status: RepositoryConfigStatusEnum, + @get:Schema(title = "文档连接", required = false) + val docUrl: String? = "" +) diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/service/RepoService.kt b/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/pojo/enums/RepositoryConfigStatusEnum.kt similarity index 80% rename from src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/service/RepoService.kt rename to src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/pojo/enums/RepositoryConfigStatusEnum.kt index 1e0fa873f91..f80c97270cd 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/service/RepoService.kt +++ b/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/pojo/enums/RepositoryConfigStatusEnum.kt @@ -25,21 +25,17 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -package com.tencent.devops.worker.common.service +package com.tencent.devops.repository.pojo.enums -import com.tencent.bkrepo.repository.pojo.token.TokenType +import io.swagger.v3.oas.annotations.media.Schema -interface RepoService { +@Schema(title = "代码库配置状态") +enum class RepositoryConfigStatusEnum { + OK, - /** - * 获取仓库token - */ - fun getRepoToken( - userId: String, - projectId: String, - repoName: String, - path: String, - type: TokenType, - expireSeconds: Long? - ): String? + // 禁用,前端不展示 + DISABLED, + + // 部署中,配置还没有准备好,如github app还没有申请 + DEPLOYING } diff --git a/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/pojo/github/GithubCheckRuns.kt b/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/pojo/github/GithubCheckRuns.kt index a1828ca31cd..0beed6a55f6 100644 --- a/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/pojo/github/GithubCheckRuns.kt +++ b/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/pojo/github/GithubCheckRuns.kt @@ -28,6 +28,8 @@ package com.tencent.devops.repository.pojo import com.fasterxml.jackson.annotation.JsonProperty +import com.tencent.devops.repository.sdk.github.pojo.CheckRunOutput +import io.swagger.v3.oas.annotations.Parameter import io.swagger.v3.oas.annotations.media.Schema @Schema(title = "check run 模型") @@ -52,5 +54,7 @@ data class GithubCheckRuns( val conclusion: String?, @JsonProperty("completed_at") @get:Schema(title = "完成于", description = "completed_at") - val completedAt: String? + val completedAt: String?, + @Parameter(description = "output", required = true) + val output: CheckRunOutput? = null ) diff --git a/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/sdk/github/pojo/CheckRunOutput.kt b/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/sdk/github/pojo/CheckRunOutput.kt index 3e3b9fd6220..7cd98c8c574 100644 --- a/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/sdk/github/pojo/CheckRunOutput.kt +++ b/src/backend/ci/core/repository/api-repository/src/main/kotlin/com/tencent/devops/repository/sdk/github/pojo/CheckRunOutput.kt @@ -1,13 +1,16 @@ package com.tencent.devops.repository.sdk.github.pojo import com.fasterxml.jackson.annotation.JsonProperty +import io.swagger.v3.oas.annotations.Parameter data class CheckRunOutput( @JsonProperty("annotations_count") - val annotationsCount: Int, + val annotationsCount: Int? = 0, @JsonProperty("annotations_url") - val annotationsUrl: String, + val annotationsUrl: String? = "", val summary: String?, - val text: String?, - val title: String? + var text: String?, + val title: String?, + @Parameter(description = "报表数据", required = true) + val reportData: Pair, MutableMap>>>? = Pair(listOf(), mutableMapOf()) ) diff --git a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/dao/RepositoryDao.kt b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/dao/RepositoryDao.kt index 6cff7dbd6c2..4efd5f778e1 100644 --- a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/dao/RepositoryDao.kt +++ b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/dao/RepositoryDao.kt @@ -545,4 +545,19 @@ class RepositoryDao { .execute() } } + + fun updateStoreRepoProject( + dslContext: DSLContext, + userId: String, + projectId: String, + repositoryId: Long + ) { + with(TRepository.T_REPOSITORY) { + dslContext.update(this) + .set(PROJECT_ID, projectId) + .set(USER_ID, userId) + .where(REPOSITORY_ID.eq(repositoryId)) + .execute() + } + } } diff --git a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/resources/ServiceRepositoryPacResourceImpl.kt b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/resources/ServiceRepositoryPacResourceImpl.kt index 95cbeedd501..88df89976bf 100644 --- a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/resources/ServiceRepositoryPacResourceImpl.kt +++ b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/resources/ServiceRepositoryPacResourceImpl.kt @@ -54,6 +54,28 @@ class ServiceRepositoryPacResourceImpl @Autowired constructor( return Result(true) } + override fun enablePac(userId: String, projectId: String, repositoryHashId: String): Result { + repositoryPacService.enablePac( + userId = userId, + projectId = projectId, + repositoryHashId = repositoryHashId + ) + return Result(true) + } + + override fun disablePac( + userId: String, + projectId: String, + repositoryHashId: String + ): Result { + repositoryPacService.disablePac( + userId = userId, + projectId = projectId, + repositoryHashId = repositoryHashId + ) + return Result(true) + } + override fun getPacRepository(externalId: String, scmType: ScmType): Result { return Result(repositoryPacService.getPacRepository(externalId = externalId, scmType = scmType)) } diff --git a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/resources/ServiceRepositoryResourceImpl.kt b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/resources/ServiceRepositoryResourceImpl.kt index 6cc4bcaab84..d7b42668f85 100644 --- a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/resources/ServiceRepositoryResourceImpl.kt +++ b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/resources/ServiceRepositoryResourceImpl.kt @@ -242,4 +242,8 @@ class ServiceRepositoryResourceImpl @Autowired constructor( ): Result> { return Result(repositoryService.getGitProjectIdByRepositoryHashId(userId, repositoryHashIdList)) } + + override fun updateStoreRepoProject(userId: String, projectId: String, repositoryId: Long): Result { + return repositoryService.updateStoreRepoProject(userId, projectId, repositoryId) + } } diff --git a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/resources/UserRepositoryConfigResourceImpl.kt b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/resources/UserRepositoryConfigResourceImpl.kt new file mode 100644 index 00000000000..228d1f966bb --- /dev/null +++ b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/resources/UserRepositoryConfigResourceImpl.kt @@ -0,0 +1,77 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.repository.resources + +import com.tencent.devops.common.api.enums.ScmType +import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.common.web.RestResource +import com.tencent.devops.common.web.utils.I18nUtil +import com.tencent.devops.repository.api.UserRepositoryConfigResource +import com.tencent.devops.repository.pojo.RepositoryConfig +import com.tencent.devops.repository.pojo.enums.RepositoryConfigStatusEnum +import com.tencent.devops.scm.config.GitConfig +import org.springframework.beans.factory.annotation.Autowired + +@RestResource +class UserRepositoryConfigResourceImpl @Autowired constructor( + private val gitConfig: GitConfig +) : UserRepositoryConfigResource { + + companion object { + // 后续需改造数据库字段 + val DOC_URL_MAP = mapOf( + ScmType.GITHUB.name to "/docs/markdown/Devops//UserGuide/Setup/guidelines-bkdevops-githubapp.md" + ) + } + + override fun list(): Result> { + // TODO 源码管理需要优化 + val managers = ScmType.values().map { + val status = when { + it == ScmType.GITHUB && gitConfig.githubClientId.isBlank() -> + RepositoryConfigStatusEnum.DEPLOYING + + it == ScmType.CODE_GIT && gitConfig.clientId.isBlank() -> + RepositoryConfigStatusEnum.DISABLED + + else -> + RepositoryConfigStatusEnum.OK + } + RepositoryConfig( + scmType = it, + name = I18nUtil.getCodeLanMessage( + messageCode = "TRIGGER_TYPE_${it.name}", + defaultMessage = it.name + ), + status = status, + docUrl = DOC_URL_MAP[it.name] ?: "" + ) + } + return Result(managers) + } +} diff --git a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/resources/UserRepositoryPacResourceImpl.kt b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/resources/UserRepositoryPacResourceImpl.kt index 7b85f8e0dde..938cf3deaa4 100644 --- a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/resources/UserRepositoryPacResourceImpl.kt +++ b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/resources/UserRepositoryPacResourceImpl.kt @@ -35,11 +35,13 @@ import com.tencent.devops.common.web.RestResource import com.tencent.devops.common.web.utils.I18nUtil import com.tencent.devops.repository.api.UserRepositoryPacResource import com.tencent.devops.repository.service.RepositoryPacService +import com.tencent.devops.scm.config.GitConfig import org.springframework.beans.factory.annotation.Autowired @RestResource class UserRepositoryPacResourceImpl @Autowired constructor( - private val repositoryPacService: RepositoryPacService + private val repositoryPacService: RepositoryPacService, + private val gitConfig: GitConfig ) : UserRepositoryPacResource { override fun getPacProjectId( @@ -122,14 +124,19 @@ class UserRepositoryPacResourceImpl @Autowired constructor( } override fun supportScmType(): Result> { - return Result(listOf(ScmType.CODE_GIT).map { - IdValue( - id = it.name, - value = I18nUtil.getCodeLanMessage( - messageCode = "TRIGGER_TYPE_${it.name}", - defaultMessage = it.name + // TODO 源码管理需要优化 + return if (gitConfig.clientId.isBlank()) { + return Result(emptyList()) + } else { + Result(listOf(ScmType.CODE_GIT).map { + IdValue( + id = it.name, + value = I18nUtil.getCodeLanMessage( + messageCode = "TRIGGER_TYPE_${it.name}", + defaultMessage = it.name + ) ) - ) - }) + }) + } } } diff --git a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/resources/scm/ServiceScmOauthResourceImpl.kt b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/resources/scm/ServiceScmOauthResourceImpl.kt index db6c5cd28de..65e818a5e21 100644 --- a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/resources/scm/ServiceScmOauthResourceImpl.kt +++ b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/resources/scm/ServiceScmOauthResourceImpl.kt @@ -90,7 +90,9 @@ class ServiceScmOauthResourceImpl @Autowired constructor(private val scmOauthSer token: String?, region: CodeSvnRegion?, userName: String?, - search: String? + search: String?, + page: Int, + pageSize: Int ): Result> { logger.info("listBranches|(projectName=$projectName, url=$url, type=$type, region=$region, username=$userName)") return Result( @@ -103,7 +105,9 @@ class ServiceScmOauthResourceImpl @Autowired constructor(private val scmOauthSer token = token, region = region, userName = userName, - search = search + search = search, + page = page, + pageSize = pageSize ) ) } diff --git a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/resources/scm/ServiceTGitResourceImpl.kt b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/resources/scm/ServiceTGitResourceImpl.kt index 586e95db190..b75022ac628 100644 --- a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/resources/scm/ServiceTGitResourceImpl.kt +++ b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/resources/scm/ServiceTGitResourceImpl.kt @@ -44,6 +44,7 @@ import com.tencent.devops.repository.service.RepositoryService import com.tencent.devops.repository.service.tgit.ITGitService import com.tencent.devops.scm.code.git.api.GitBranch import com.tencent.devops.scm.enums.GitAccessLevelEnum +import com.tencent.devops.scm.pojo.ChangeFileInfo import com.tencent.devops.scm.pojo.GitFileInfo import javax.servlet.http.HttpServletResponse import org.springframework.beans.factory.annotation.Autowired @@ -170,4 +171,30 @@ class ServiceTGitResourceImpl @Autowired constructor( ) ) } + + override fun getChangeFileList( + token: String, + tokenType: TokenTypeEnum, + gitProjectId: String, + from: String, + to: String, + straight: Boolean?, + page: Int, + pageSize: Int, + url: String + ): Result> { + return Result( + gitService.getChangeFileList( + tokenType = tokenType, + gitProjectId = gitProjectId, + token = token, + from = from, + to = to, + straight = straight, + page = page, + pageSize = pageSize, + url = url + ) + ) + } } diff --git a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/RepositoryService.kt b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/RepositoryService.kt index 9f792dde330..d5ed7d1f034 100644 --- a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/RepositoryService.kt +++ b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/RepositoryService.kt @@ -87,15 +87,15 @@ import com.tencent.devops.scm.pojo.GitCommit import com.tencent.devops.scm.pojo.GitProjectInfo import com.tencent.devops.scm.pojo.GitRepositoryDirItem import com.tencent.devops.scm.pojo.GitRepositoryResp +import java.time.LocalDateTime +import java.util.Base64 +import javax.ws.rs.NotFoundException import org.jooq.DSLContext import org.jooq.impl.DSL import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.beans.factory.annotation.Value import org.springframework.stereotype.Service -import java.time.LocalDateTime -import java.util.Base64 -import javax.ws.rs.NotFoundException @Service @Suppress("ALL") @@ -1346,6 +1346,16 @@ class RepositoryService @Autowired constructor( } } + fun updateStoreRepoProject(userId: String, projectId: String, repositoryId: Long): Result { + repositoryDao.updateStoreRepoProject( + dslContext = dslContext, + userId = userId, + projectId = projectId, + repositoryId = repositoryId + ) + return Result(true) + } + fun getGitProjectIdByRepositoryHashId(userId: String, repositoryHashIdList: List): List { return repositoryDao.getGitProjectIdByRepositoryHashId(dslContext, repositoryHashIdList) } diff --git a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/github/GithubService.kt b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/github/GithubService.kt index 58359a9dca0..d7c324571b6 100644 --- a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/github/GithubService.kt +++ b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/github/GithubService.kt @@ -61,6 +61,7 @@ import com.tencent.devops.repository.pojo.github.GithubTag import com.tencent.devops.repository.sdk.github.request.GetRepositoryContentRequest import com.tencent.devops.repository.sdk.github.service.GithubRepositoryService import com.tencent.devops.repository.sdk.github.service.GithubUserService +import com.tencent.devops.repository.utils.scm.QualityUtils import com.tencent.devops.scm.config.GitConfig import com.tencent.devops.scm.exception.GithubApiException import com.tencent.devops.scm.pojo.Project @@ -137,7 +138,14 @@ class GithubService @Autowired constructor( ) { logger.warn("conclusion and completedAt must be null or not null together") } - + checkRuns.output?.let { + if (it.reportData?.second?.isNotEmpty() == true) { + it.text = QualityUtils.getQualityReport( + titleData = it.reportData!!.first, + resultData = it.reportData!!.second + ) + } + } val body = objectMapper.writeValueAsString(checkRuns) val request = buildPatch(token, "repos/$projectName/check-runs/$checkRunId", body) val operation = getMessageByLocale(OPERATION_UPDATE_CHECK_RUNS) diff --git a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/scm/IScmOauthService.kt b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/scm/IScmOauthService.kt index 9117238a46a..799eb38203b 100644 --- a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/scm/IScmOauthService.kt +++ b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/scm/IScmOauthService.kt @@ -63,7 +63,9 @@ interface IScmOauthService { token: String?, region: CodeSvnRegion?, userName: String?, - search: String? = null + search: String? = null, + page: Int = 1, + pageSize: Int = 20 ): List fun listTags( diff --git a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/scm/ScmOauthService.kt b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/scm/ScmOauthService.kt index 19ded758298..0d22aa0ea28 100644 --- a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/scm/ScmOauthService.kt +++ b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/scm/ScmOauthService.kt @@ -94,7 +94,9 @@ class ScmOauthService @Autowired constructor( token: String?, region: CodeSvnRegion?, userName: String?, - search: String? + search: String?, + page: Int, + pageSize: Int ): List { logger.info("[$projectName|$url|$type|$userName] Start to list the branches") val startEpoch = System.currentTimeMillis() @@ -110,7 +112,7 @@ class ScmOauthService @Autowired constructor( region = region, userName = userName, event = null - ).getBranches(search = search) + ).getBranches(search = search, page, pageSize) } finally { logger.info("It took ${System.currentTimeMillis() - startEpoch}ms to list branches") } diff --git a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/tgit/ITGitService.kt b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/tgit/ITGitService.kt index 993e3ba4d89..369e93c8935 100644 --- a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/tgit/ITGitService.kt +++ b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/tgit/ITGitService.kt @@ -37,6 +37,7 @@ import com.tencent.devops.repository.pojo.oauth.GitToken import com.tencent.devops.scm.code.git.api.GitBranch import com.tencent.devops.scm.code.git.api.GitTag import com.tencent.devops.scm.enums.GitAccessLevelEnum +import com.tencent.devops.scm.pojo.ChangeFileInfo import com.tencent.devops.scm.pojo.GitFileInfo import javax.servlet.http.HttpServletResponse @@ -95,4 +96,16 @@ interface ITGitService { owned: Boolean?, minAccessLevel: GitAccessLevelEnum? ): List + + fun getChangeFileList( + token: String, + tokenType: TokenTypeEnum, + gitProjectId: String, + from: String, + to: String, + straight: Boolean? = false, + page: Int, + pageSize: Int, + url: String + ): List } diff --git a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/tgit/TGitService.kt b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/tgit/TGitService.kt index 7b0eaa8ec5a..f28867db192 100644 --- a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/tgit/TGitService.kt +++ b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/service/tgit/TGitService.kt @@ -50,7 +50,9 @@ import com.tencent.devops.scm.code.git.api.GitTag import com.tencent.devops.scm.code.git.api.GitTagCommit import com.tencent.devops.scm.config.GitConfig import com.tencent.devops.scm.enums.GitAccessLevelEnum +import com.tencent.devops.scm.pojo.ChangeFileInfo import com.tencent.devops.scm.pojo.GitFileInfo +import com.tencent.devops.scm.utils.code.git.GitUtils import java.net.URLEncoder import javax.servlet.http.HttpServletResponse import javax.ws.rs.core.Response @@ -77,7 +79,7 @@ class TGitService @Autowired constructor( "${gitConfig.tGitUrl}/oauth/token?client_id=${gitConfig.tGitClientId}" + "&client_secret=${gitConfig.tGitClientSecret}&code=$code" + "&grant_type=authorization_code&" + - "redirect_uri=${URLEncoder.encode(gitConfig.tGitWebhookUrl, "utf-8")}" + "redirect_uri=${urlEncode(gitConfig.tGitWebhookUrl)}" logger.info("getToken url>> $tokenUrl") val request = Request.Builder() .url(tokenUrl) @@ -150,7 +152,7 @@ class TGitService @Autowired constructor( val pageNotNull = page ?: 1 val pageSizeNotNull = pageSize ?: 20 logger.info("start to get the $userId's $repository branch by accessToken") - val repoId = URLEncoder.encode(repository, "utf-8") + val repoId = urlEncode(repository) val url = "${gitConfig.tGitApiUrl}/projects/$repoId/repository/branches" + "?access_token=$accessToken&page=$pageNotNull&per_page=$pageSizeNotNull" + if (search != null) { @@ -213,7 +215,7 @@ class TGitService @Autowired constructor( val pageNotNull = page ?: 1 val pageSizeNotNull = pageSize ?: 20 logger.info("start to get the $userId's $repository tag by page: $pageNotNull pageSize: $pageSizeNotNull") - val repoId = URLEncoder.encode(repository, "utf-8") + val repoId = urlEncode(repository) val url = "${gitConfig.tGitApiUrl}/projects/$repoId/repository/tags" + "?access_token=$accessToken&page=$pageNotNull&per_page=$pageSizeNotNull" val res = mutableListOf() @@ -274,8 +276,8 @@ class TGitService @Autowired constructor( ): String { val startEpoch = System.currentTimeMillis() try { - var url = "${gitConfig.tGitApiUrl}/projects/${URLEncoder.encode(repoName, "UTF-8")}/repository/blobs/" + - "${URLEncoder.encode(ref, "UTF-8")}?filepath=${URLEncoder.encode(filePath, "UTF-8")}" + var url = "${gitConfig.tGitApiUrl}/projects/${urlEncode(repoName)}/repository/blobs/" + + "${urlEncode(ref)}?filepath=${urlEncode(filePath)}" logger.info("[$repoName|$filePath|$authType|$ref] Start to get the git file content from $url") val request = if (authType == RepoAuthType.OAUTH) { @@ -317,9 +319,9 @@ class TGitService @Autowired constructor( ) { val startEpoch = System.currentTimeMillis() try { - val url = "${gitConfig.gitApiUrl}/projects/${URLEncoder.encode(repoName, "UTF-8")}/repository/" + - "blobs/${URLEncoder.encode(ref, "UTF-8")}?" + - "filepath=${URLEncoder.encode(filePath, "UTF-8")}&access_token=$token" + val url = "${gitConfig.gitApiUrl}/projects/${urlEncode(repoName)}/repository/" + + "blobs/${urlEncode(ref)}?" + + "filepath=${urlEncode(filePath)}&access_token=$token" val request = Request.Builder() .url(url) @@ -353,16 +355,16 @@ class TGitService @Autowired constructor( try { val url = StringBuilder( "${gitConfig.tGitApiUrl}/projects/" + - "${URLEncoder.encode(gitProjectId, "UTF-8")}/repository/tree" + "${urlEncode(gitProjectId)}/repository/tree" ) setToken(tokenType, url, token) with(url) { append( - "&path=${URLEncoder.encode(path, "UTF-8")}" + "&path=${urlEncode(path)}" ) append( if (!ref.isNullOrBlank()) { - "&ref_name=${URLEncoder.encode(ref, "UTF-8")}" + "&ref_name=${urlEncode(ref)}" } else { "" } @@ -457,6 +459,55 @@ class TGitService @Autowired constructor( return sb.toString() } + override fun getChangeFileList( + token: String, + tokenType: TokenTypeEnum, + gitProjectId: String, + from: String, + to: String, + straight: Boolean?, + page: Int, + pageSize: Int, + url: String + ): List { + val host = GitUtils.getGitApiUrl(apiUrl = gitConfig.tGitApiUrl, repoUrl = url) + val apiUrl = StringBuilder("$host/projects/${urlEncode(gitProjectId)}/" + + "repository/compare/changed_files/list") + setToken(tokenType, apiUrl, token) + val requestUrl = apiUrl.toString().addParams( + mapOf( + "from" to from, + "to" to to, + "straight" to straight, + "page" to page, + "pageSize" to pageSize + ) + ) + val res = mutableListOf() + val request = Request.Builder() + .url(requestUrl) + .get() + .build() + var result = res.toList() + logger.info("getChangeFileList: $requestUrl") + OkhttpUtils.doHttp(request).use { response -> + if (!response.isSuccessful) { + throw RemoteServiceException( + httpStatus = response.code, + errorMessage = "(${response.code})${response.message}" + ) + } + val data = response.body?.string() ?: return@use + val repoList = JsonParser().parse(data).asJsonArray + if (!repoList.isJsonNull) { + result = JsonUtil.to(data, object : TypeReference>() {}) + } + } + return result + } + + private fun urlEncode(s: String) = URLEncoder.encode(s, "UTF-8") + companion object { private val logger = LoggerFactory.getLogger(TGitService::class.java) private const val PAGE_SIZE = 100 diff --git a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/utils/scm/QualityUtils.kt b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/utils/scm/QualityUtils.kt index 01e483ff90f..aa54731930c 100644 --- a/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/utils/scm/QualityUtils.kt +++ b/src/backend/ci/core/repository/biz-repository/src/main/kotlin/com/tencent/devops/repository/utils/scm/QualityUtils.kt @@ -37,10 +37,14 @@ object QualityUtils { val url = titleData[4] val pipelineNameTitle = titleData[5] val ruleName = titleData[6] - + val pipelineLinkElement = if (url.isBlank()) { + pipelineName + } else { + "$pipelineName" + } val title = "" + "" + - "" + + "" + "" + "" + "" + diff --git a/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/common/OpStoreComponentResource.kt b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/common/OpStoreComponentResource.kt index 7a9ebde040f..a48548ca6d1 100644 --- a/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/common/OpStoreComponentResource.kt +++ b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/common/OpStoreComponentResource.kt @@ -26,16 +26,19 @@ */ package com.tencent.devops.store.api.common +import com.tencent.devops.common.api.annotation.BkInterfaceI18n import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID_DEFAULT_VALUE import com.tencent.devops.common.api.pojo.Page import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.web.annotation.BkField import com.tencent.devops.common.web.constant.BkStyleEnum +import com.tencent.devops.store.pojo.common.InstalledPkgShaContentRequest import com.tencent.devops.store.pojo.common.MyStoreComponent import com.tencent.devops.store.pojo.common.StoreBaseInfoUpdateRequest import com.tencent.devops.store.pojo.common.StoreDetailInfo import com.tencent.devops.store.pojo.common.enums.StoreSortTypeEnum +import com.tencent.devops.store.pojo.common.enums.StoreTypeEnum import com.tencent.devops.store.pojo.common.publication.StoreApproveReleaseRequest import com.tencent.devops.store.pojo.common.publication.StoreOfflineRequest import com.tencent.devops.store.pojo.common.version.StoreDeskVersionItem @@ -77,6 +80,10 @@ interface OpStoreComponentResource { @Operation(summary = "获取组件列表") @GET @Path("/types/{storeType}/component/list") + @BkInterfaceI18n( + keyPrefixNames = ["{data.records[*].storeType}", "{data.records[*].storeCode}", "{data.records[*].version}", + "releaseInfo"] + ) @Suppress("LongParameterList") fun listComponents( @Parameter(description = "userId", required = true) @@ -120,6 +127,10 @@ interface OpStoreComponentResource { @Operation(summary = "根据组件标识获取组件版本列表") @GET @Path("/types/{storeType}/codes/{storeCode}/component/version/list") + @BkInterfaceI18n( + keyPrefixNames = ["{data.records[*].storeType}", "{data.records[*].storeCode}", "{data.records[*].version}", + "releaseInfo"] + ) fun getComponentVersionsByCode( @Parameter(description = "userId", required = true) @HeaderParam(AUTH_HEADER_USER_ID) @@ -145,6 +156,9 @@ interface OpStoreComponentResource { @Operation(summary = "根据组件ID获取组件详情") @GET @Path("/types/{storeType}/ids/{storeId}/component/detail") + @BkInterfaceI18n( + keyPrefixNames = ["{data.storeType}", "{data.storeCode}", "{data.version}", "releaseInfo"] + ) fun getComponentDetailInfoById( @Parameter(description = "userId", required = true) @HeaderParam(AUTH_HEADER_USER_ID) @@ -206,4 +220,26 @@ interface OpStoreComponentResource { @Valid storeOfflineRequest: StoreOfflineRequest ): Result + + @Operation(summary = "更新组件已安装包sha1摘要值") + @PUT + @Path("/types/{storeType}/codes/{storeCode}/versions/{version}/component/installed/pkg/sha/update") + fun updateComponentInstalledPkgShaContent( + @Parameter(description = "用户ID", required = true, example = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @Parameter(description = "组件类型", required = true) + @PathParam("storeType") + @BkField(patternStyle = BkStyleEnum.CODE_STYLE) + storeType: StoreTypeEnum, + @Parameter(description = "组件代码", required = true) + @PathParam("storeCode") + @BkField(patternStyle = BkStyleEnum.CODE_STYLE) + storeCode: String, + @Parameter(description = "组件版本", required = true) + @PathParam("version") + version: String, + @Parameter(description = "更新组件已安装包sha1摘要值请求报文", required = true) + installedPkgShaContentRequest: InstalledPkgShaContentRequest + ): Result } diff --git a/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/common/OpStoreProjectResource.kt b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/common/OpStoreProjectResource.kt new file mode 100644 index 00000000000..a2405c201cd --- /dev/null +++ b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/common/OpStoreProjectResource.kt @@ -0,0 +1,60 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.store.api.common + +import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID +import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID_DEFAULT_VALUE +import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.store.pojo.common.StoreProjectInfo +import io.swagger.v3.oas.annotations.Operation +import io.swagger.v3.oas.annotations.Parameter +import io.swagger.v3.oas.annotations.tags.Tag +import javax.ws.rs.Consumes +import javax.ws.rs.HeaderParam +import javax.ws.rs.PUT +import javax.ws.rs.Path +import javax.ws.rs.Produces +import javax.ws.rs.core.MediaType + +@Tag(name = "OP_STORE_PROJECT", description = "OP-组件-项目") +@Path("/op/store/project") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +interface OpStoreProjectResource { + + @Operation(summary = "更新组件关联初始化项目信息") + @PUT + @Path("/relevancy/update") + fun updateStoreInitProject( + @Parameter(description = "用户ID", required = true, example = AUTH_HEADER_USER_ID_DEFAULT_VALUE) + @HeaderParam(AUTH_HEADER_USER_ID) + userId: String, + @Parameter(description = "组件关联初始化项目信息", required = true) + storeProjectInfo: StoreProjectInfo + ): Result +} diff --git a/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/common/OpenStoreResource.kt b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/common/OpenStoreResource.kt index cb805033123..4784500b50f 100644 --- a/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/common/OpenStoreResource.kt +++ b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/common/OpenStoreResource.kt @@ -30,9 +30,9 @@ package com.tencent.devops.store.api.common import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_BK_TOKEN import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.store.pojo.common.enums.StoreTypeEnum -import io.swagger.v3.oas.annotations.tags.Tag import io.swagger.v3.oas.annotations.Operation import io.swagger.v3.oas.annotations.Parameter +import io.swagger.v3.oas.annotations.tags.Tag import javax.ws.rs.Consumes import javax.ws.rs.GET import javax.ws.rs.HeaderParam diff --git a/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/common/ServiceStoreComponentResource.kt b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/common/ServiceStoreComponentResource.kt index dc7a6e220a3..80613600e52 100644 --- a/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/common/ServiceStoreComponentResource.kt +++ b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/common/ServiceStoreComponentResource.kt @@ -63,8 +63,7 @@ interface ServiceStoreComponentResource { @Path("/types/{storeType}/component/main/page/list") @GET @BkInterfaceI18n( - keyPrefixNames = [ - "{data.records[*].storeType}", "{data.records[*].storeCode}", "{data.records[*].version}", + keyPrefixNames = ["{data[*].records[*].type}", "{data[*].records[*].code}", "{data[*].records[*].version}", "releaseInfo" ] ) @@ -98,7 +97,7 @@ interface ServiceStoreComponentResource { @GET @BkInterfaceI18n( keyPrefixNames = [ - "{data.records[*].storeType}", "{data.records[*].storeCode}", "{data.records[*].version}", + "{data.records[*].type}", "{data.records[*].code}", "{data.records[*].version}", "releaseInfo" ] ) @@ -173,7 +172,7 @@ interface ServiceStoreComponentResource { @Path("/types/{storeType}/ids/{storeId}/component/detail") @BkInterfaceI18n( keyPrefixNames = [ - "{data.records[*].storeType}", "{data.records[*].storeCode}", "{data.records[*].version}", + "{data.storeType}", "{data.storeCode}", "{data.version}", "releaseInfo" ] ) diff --git a/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/common/UserSensitiveApiResource.kt b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/common/UserSensitiveApiResource.kt index d3ab4d2d926..285ca08e967 100644 --- a/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/common/UserSensitiveApiResource.kt +++ b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/common/UserSensitiveApiResource.kt @@ -68,9 +68,9 @@ interface UserSensitiveApiResource { @Parameter(description = "组件标识", required = true) @PathParam("storeCode") storeCode: String, - @Parameter(description = "组件标识", required = true) + @Parameter(description = "开发语言", required = true) @QueryParam("language") - language: String + language: String = "" ): Result> @Operation(summary = "敏感API申请") diff --git a/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/common/UserStoreComponentQueryResource.kt b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/common/UserStoreComponentQueryResource.kt index 11b7e508bb7..c6cb9267aed 100644 --- a/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/common/UserStoreComponentQueryResource.kt +++ b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/api/common/UserStoreComponentQueryResource.kt @@ -119,8 +119,7 @@ interface UserStoreComponentQueryResource { @GET @Path("/types/{storeType}/ids/{storeId}/component/detail") @BkInterfaceI18n( - keyPrefixNames = ["{data.records[*].storeType}", "{data.records[*].storeCode}", "{data.records[*].version}", - "releaseInfo"] + keyPrefixNames = ["{data.storeType}", "{data.storeCode}", "{data.version}", "releaseInfo"] ) fun getComponentDetailInfoById( @Parameter(description = "userId", required = true) @@ -140,8 +139,7 @@ interface UserStoreComponentQueryResource { @GET @Path("/types/{storeType}/codes/{storeCode}/component/detail") @BkInterfaceI18n( - keyPrefixNames = ["{data.records[*].storeType}", "{data.records[*].storeCode}", "{data.records[*].version}", - "releaseInfo"] + keyPrefixNames = ["{data.storeType}", "{data.storeCode}", "{data.version}", "releaseInfo"] ) fun getComponentDetailInfoByCode( @Parameter(description = "userId", required = true) @@ -161,8 +159,9 @@ interface UserStoreComponentQueryResource { @Path("/types/{storeType}/component/main/page/list") @GET @BkInterfaceI18n( - keyPrefixNames = ["{data.records[*].storeType}", "{data.records[*].storeCode}", "{data.records[*].version}", - "releaseInfo"] + keyPrefixNames = ["{data[*].records[*].type}", "{data[*].records[*].code}", "{data[*].records[*].version}", + "releaseInfo" + ] ) @Suppress("LongParameterList") fun getMainPageComponents( @@ -193,8 +192,10 @@ interface UserStoreComponentQueryResource { @Path("/types/{storeType}/component/list") @GET @BkInterfaceI18n( - keyPrefixNames = ["{data.records[*].storeType}", "{data.records[*].storeCode}", "{data.records[*].version}", - "releaseInfo"] + keyPrefixNames = [ + "{data.records[*].type}", "{data.records[*].code}", "{data.records[*].version}", + "releaseInfo" + ] ) @Suppress("LongParameterList") fun queryComponents( diff --git a/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/constant/StoreMessageCode.kt b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/constant/StoreMessageCode.kt index 878fe7caff4..377c9b2e4fa 100644 --- a/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/constant/StoreMessageCode.kt +++ b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/constant/StoreMessageCode.kt @@ -125,6 +125,7 @@ object StoreMessageCode { const val USER_SERVICE_PROJECT_NOT_PERMISSION = "2120408" // 研发商店：选中调试项目无创建流水线权限 const val USER_SERVICE_NOT_EXIST = "2120409" // 研发商店：扩展服务不存在{0} const val USER_ITEM_SERVICE_USED_IS_NOT_ALLOW_DELETE = "2120410" // 研发商店：扩展点下还有可用的扩展服务，不能删除 + const val USER_SERVICE_NOT_DEPLOY = "2120411" // 研发商店:用户扩展服务未部署 const val USER_PRAISE_IS_INVALID = "2120901" // 研发商店：你已点赞过 const val USER_PROJECT_IS_NOT_ALLOW_INSTALL = "2120902" // 研发商店：你没有权限将组件安装到项目：{0} diff --git a/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/pojo/common/InstalledPkgShaContentRequest.kt b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/pojo/common/InstalledPkgShaContentRequest.kt new file mode 100644 index 00000000000..3cf63443530 --- /dev/null +++ b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/pojo/common/InstalledPkgShaContentRequest.kt @@ -0,0 +1,40 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.store.pojo.common + +import io.swagger.v3.oas.annotations.media.Schema + +@Schema(title = "更新组件已安装包sha1摘要值请求报文") +data class InstalledPkgShaContentRequest( + @get:Schema(title = "已安装包sha1摘要值", required = true) + val installedPkgShaContent: String, + @get:Schema(title = "操作系统名称", required = false) + val osName: String? = null, + @get:Schema(title = "操作系统CPU架构", required = false) + val osArch: String? = null +) diff --git a/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/pojo/common/MyStoreComponent.kt b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/pojo/common/MyStoreComponent.kt index 7b68809be3f..9d03e9af195 100644 --- a/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/pojo/common/MyStoreComponent.kt +++ b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/pojo/common/MyStoreComponent.kt @@ -27,6 +27,8 @@ package com.tencent.devops.store.pojo.common +import com.tencent.devops.common.api.annotation.BkFieldI18n +import com.tencent.devops.common.api.enums.I18nSourceEnum import io.swagger.v3.oas.annotations.media.Schema @Schema(title = "研发商店-我的组件信息") @@ -38,6 +40,7 @@ data class MyStoreComponent( @get:Schema(title = "store组件类型", required = true) val storeType: String, @get:Schema(title = "store组件名称", required = true) + @BkFieldI18n(source = I18nSourceEnum.DB) val name: String, @get:Schema(title = "开发语言", required = false) val language: String? = null, diff --git a/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/pojo/common/StoreProjectInfo.kt b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/pojo/common/StoreProjectInfo.kt new file mode 100644 index 00000000000..5e3dae00fb5 --- /dev/null +++ b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/pojo/common/StoreProjectInfo.kt @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.store.pojo.common + +import com.tencent.devops.store.pojo.common.enums.StoreTypeEnum +import io.swagger.v3.oas.annotations.media.Schema + +@Schema(title = "组件关联初始化项目信息") +data class StoreProjectInfo( + @get:Schema(title = "组件负责人", required = true) + val userId: String, + @get:Schema(title = "组件代码", required = true) + val storeCode: String, + @get:Schema(title = "项目ID", required = true) + val projectId: String, + @get:Schema(title = "组件类型", required = true) + val storeType: StoreTypeEnum +) diff --git a/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/pojo/common/sensitive/SensitiveApiApplyReq.kt b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/pojo/common/sensitive/SensitiveApiApplyReq.kt index 0f17a6ee30e..4052cb7d969 100644 --- a/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/pojo/common/sensitive/SensitiveApiApplyReq.kt +++ b/src/backend/ci/core/store/api-store/src/main/kotlin/com/tencent/devops/store/pojo/common/sensitive/SensitiveApiApplyReq.kt @@ -35,6 +35,6 @@ data class SensitiveApiApplyReq( val apiNameList: List, @get:Schema(title = "申请说明", required = true) val applyDesc: String, - @get:Schema(title = "插件使用的语言", required = true) - val language: String + @get:Schema(title = "开发语言", required = true) + val language: String = "" ) diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/atom/dao/AtomCommonDao.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/atom/dao/AtomCommonDao.kt index 70b30d8d646..7682cd6f102 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/atom/dao/AtomCommonDao.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/atom/dao/AtomCommonDao.kt @@ -184,4 +184,13 @@ class AtomCommonDao : AbstractStoreCommonDao() { } } } + + override fun getStoreRepoHashIdByCode(dslContext: DSLContext, storeCode: String): String? { + with(TAtom.T_ATOM) { + return dslContext.select(REPOSITORY_HASH_ID) + .from(this) + .where(ATOM_CODE.eq(storeCode)) + .fetchAny()?.into(String::class.java) + } + } } diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/atom/service/impl/AtomMemberServiceImpl.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/atom/service/impl/AtomMemberServiceImpl.kt index 42dd16ada14..884abf97d38 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/atom/service/impl/AtomMemberServiceImpl.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/atom/service/impl/AtomMemberServiceImpl.kt @@ -150,7 +150,7 @@ abstract class AtomMemberServiceImpl : StoreMemberServiceImpl() { abstract fun deleteRepoMember(userId: String, username: String, repositoryHashId: String): Result - override fun getStoreName(storeCode: String): String { + override fun getStoreName(storeCode: String, storeType: StoreTypeEnum): String { return marketAtomDao.getLatestAtomByCode(dslContext, storeCode)?.name ?: "" } } diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/atom/service/impl/AtomPropServiceImpl.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/atom/service/impl/AtomPropServiceImpl.kt index 041334f0853..0f9255c770f 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/atom/service/impl/AtomPropServiceImpl.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/atom/service/impl/AtomPropServiceImpl.kt @@ -91,7 +91,7 @@ class AtomPropServiceImpl @Autowired constructor( logoUrl = logoUrl?.let { StoreDecorateFactory.get(StoreDecorateFactory.Kind.HOST)?.decorate(logoUrl) as? String } - logoUrl = RegexUtils.trimProtocol(atomPropRecord[tAtom.LOGO_URL]) + logoUrl = RegexUtils.trimProtocol(logoUrl) val atomProp = AtomProp( atomCode = atomCode, os = JsonUtil.to(atomPropRecord[tAtom.OS], object : TypeReference>() {}), diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/atom/service/impl/JavaAtomBusHandleHandleServiceImpl.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/atom/service/impl/JavaAtomBusHandleHandleServiceImpl.kt index 3c7ff5e0426..88b7a0ab3b9 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/atom/service/impl/JavaAtomBusHandleHandleServiceImpl.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/atom/service/impl/JavaAtomBusHandleHandleServiceImpl.kt @@ -49,12 +49,21 @@ class JavaAtomBusHandleHandleServiceImpl : AtomBusHandleService { if (reqTarget.isNullOrBlank()) { return target } - val javaPath = reqTarget.substringBefore("-jar").trim() - // 获取插件配置的JVM指令 - val jvmOptions = target.substringAfter("java").substringBefore("-jar").trim() - // 获取插件jar包路径 - val jarPath = reqTarget.substringAfter("-jar").trim() + // 获取启动命令的java路径 + val javaPath = reqTarget.substringBefore(" -").trim() + // 获取启动命令的前缀路径 + val prefixPath = target.substringBefore(getJarName(target)).substringAfter(target.substringBefore(" -")).trim() + // 获取插件jar包名称 + val jarName = getJarName(reqTarget) + // 获取启动命令的后缀路径 + val suffixPath = target.substringAfter(".jar").trim() + return "$javaPath $prefixPath $jarName $suffixPath".trim() + } - return "$javaPath $jvmOptions -jar $jarPath" + private fun getJarName(target: String): String { + val regex = Regex("(\\S+\\.jar)") + val matchResult = regex.find(target) + val jarName = matchResult?.value ?: "" + return jarName } } diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/atom/service/impl/MarketAtomCommonServiceImpl.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/atom/service/impl/MarketAtomCommonServiceImpl.kt index 63596bb39a0..39b87f62679 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/atom/service/impl/MarketAtomCommonServiceImpl.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/atom/service/impl/MarketAtomCommonServiceImpl.kt @@ -49,6 +49,7 @@ import com.tencent.devops.store.atom.dao.AtomDao import com.tencent.devops.store.atom.dao.MarketAtomDao import com.tencent.devops.store.atom.dao.MarketAtomEnvInfoDao import com.tencent.devops.store.atom.dao.MarketAtomVersionLogDao +import com.tencent.devops.store.atom.factory.AtomBusHandleFactory import com.tencent.devops.store.atom.service.MarketAtomCommonService import com.tencent.devops.store.common.dao.StoreProjectRelDao import com.tencent.devops.store.common.service.StoreCommonService @@ -433,6 +434,7 @@ class MarketAtomCommonServiceImpl : MarketAtomCommonService { } else { runtimeVersion } + val atomBusHandleService = AtomBusHandleFactory.createAtomBusHandleService(language) if (null != osList) { val osDefaultEnvNumMap = mutableMapOf() osList.forEach { osExecutionInfoMap -> diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/atom/service/impl/SampleAtomReleaseServiceImpl.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/atom/service/impl/SampleAtomReleaseServiceImpl.kt index 9868e7a4016..e1d7949f49d 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/atom/service/impl/SampleAtomReleaseServiceImpl.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/atom/service/impl/SampleAtomReleaseServiceImpl.kt @@ -41,6 +41,7 @@ import com.tencent.devops.common.api.constant.SUCCESS import com.tencent.devops.common.api.constant.TEST import com.tencent.devops.common.api.constant.UNDO import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.common.redis.RedisLock import com.tencent.devops.common.web.utils.I18nUtil import com.tencent.devops.store.atom.service.SampleAtomReleaseService import com.tencent.devops.store.constant.StoreMessageCode @@ -49,6 +50,7 @@ import com.tencent.devops.store.pojo.atom.AtomReleaseRequest import com.tencent.devops.store.pojo.atom.MarketAtomCreateRequest import com.tencent.devops.store.pojo.atom.MarketAtomUpdateRequest import com.tencent.devops.store.pojo.atom.enums.AtomStatusEnum +import com.tencent.devops.store.pojo.common.STORE_LATEST_TEST_FLAG_KEY_PREFIX import com.tencent.devops.store.pojo.common.publication.ReleaseProcessItem import com.tencent.devops.store.pojo.common.enums.StoreTypeEnum import org.jooq.DSLContext @@ -89,7 +91,24 @@ class SampleAtomReleaseServiceImpl : SampleAtomReleaseService, AtomReleaseServic branch: String?, validOsNameFlag: Boolean?, validOsArchFlag: Boolean? - ) = Unit + ) { + val record = marketAtomDao.getAtomRecordById(dslContext, atomId) + record?.let { + RedisLock( + redisOperation, + "$STORE_LATEST_TEST_FLAG_KEY_PREFIX:${record.atomCode}", + 60L + ).use { redisLock -> + redisLock.lock() + marketAtomDao.setupAtomLatestTestFlag( + dslContext = dslContext, + userId = userId, + atomCode = record.atomCode, + atomId = atomId + ) + } + } + } override fun validateUpdateMarketAtomReq( userId: String, diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/dao/AbstractStoreCommonDao.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/dao/AbstractStoreCommonDao.kt index dba41ae81cf..2321e79dff1 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/dao/AbstractStoreCommonDao.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/dao/AbstractStoreCommonDao.kt @@ -75,4 +75,6 @@ abstract class AbstractStoreCommonDao { storeCode: String, storeStatus: Byte? = null ): StoreBaseInfo? + + abstract fun getStoreRepoHashIdByCode(dslContext: DSLContext, storeCode: String): String? } diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/dao/StoreBaseEnvExtQueryDao.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/dao/StoreBaseEnvExtQueryDao.kt index a1e2a3c1a86..a7f2dc8d7f3 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/dao/StoreBaseEnvExtQueryDao.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/dao/StoreBaseEnvExtQueryDao.kt @@ -29,6 +29,7 @@ package com.tencent.devops.store.common.dao import com.tencent.devops.model.store.tables.TStoreBaseEnvExt import com.tencent.devops.model.store.tables.records.TStoreBaseEnvExtRecord +import org.jooq.Condition import org.jooq.DSLContext import org.jooq.Result import org.springframework.stereotype.Repository @@ -47,10 +48,16 @@ class StoreBaseEnvExtQueryDao { fun getBaseExtEnvsByEnvId( dslContext: DSLContext, - envId: String + envId: String, + fieldName: String? = null ): Result? { return with(TStoreBaseEnvExt.T_STORE_BASE_ENV_EXT) { - dslContext.selectFrom(this).where(ENV_ID.eq(envId)).fetch() + val conditions = mutableListOf() + conditions.add(ENV_ID.eq(envId)) + if (!fieldName.isNullOrBlank()) { + conditions.add(FIELD_NAME.eq(fieldName)) + } + dslContext.selectFrom(this).where(conditions).fetch() } } } diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/dao/StoreBaseQueryDao.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/dao/StoreBaseQueryDao.kt index 4b26c009889..346e05033fe 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/dao/StoreBaseQueryDao.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/dao/StoreBaseQueryDao.kt @@ -139,12 +139,12 @@ class StoreBaseQueryDao { return with(TStoreBase.T_STORE_BASE) { val conditions = mutableListOf(STORE_CODE.`in`(storeCodes)) conditions.add(STORE_TYPE.eq(storeType.type.toByte())) + val testStatusEnumList = listOf( + StoreStatusEnum.TESTING, + StoreStatusEnum.AUDITING + ) if (testComponentFlag) { - val statusEnumList = listOf( - StoreStatusEnum.TESTING, - StoreStatusEnum.AUDITING - ) - conditions.add(STATUS.`in`(statusEnumList)) + conditions.add(STATUS.`in`(testStatusEnumList)) val subQuery = dslContext.select( STORE_CODE, STORE_TYPE, @@ -163,6 +163,7 @@ class StoreBaseQueryDao { .fetch() } else { conditions.add(LATEST_FLAG.eq(true)) + conditions.add(STATUS.notIn(testStatusEnumList)) dslContext.selectFrom(this) .where(conditions) .fetch() @@ -221,6 +222,25 @@ class StoreBaseQueryDao { } } + fun getComponentId( + dslContext: DSLContext, + storeCode: String, + version: String, + storeType: StoreTypeEnum + ): String? { + return with(TStoreBase.T_STORE_BASE) { + val conditions = mutableListOf() + conditions.add(STORE_TYPE.eq(storeType.type.toByte())) + conditions.add(STORE_CODE.eq(storeCode)) + conditions.add(VERSION.like(VersionUtils.generateQueryVersion(version))) + dslContext.select(ID).from(this) + .where(conditions) + .orderBy(CREATE_TIME.desc()) + .limit(1) + .fetchOne(0, String::class.java) + } + } + fun countByCondition( dslContext: DSLContext, storeType: StoreTypeEnum, diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/dao/StorePipelineBuildRelDao.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/dao/StorePipelineBuildRelDao.kt index 1919b213ffb..e89d2902098 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/dao/StorePipelineBuildRelDao.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/dao/StorePipelineBuildRelDao.kt @@ -84,4 +84,12 @@ class StorePipelineBuildRelDao { .execute() } } + + fun deleteStorePipelineBuildRelByPipelineId(dslContext: DSLContext, pipelineId: String) { + with(TStorePipelineBuildRel.T_STORE_PIPELINE_BUILD_REL) { + dslContext.deleteFrom(this) + .where(PIPELINE_ID.eq(pipelineId)) + .execute() + } + } } diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/dao/StorePipelineRelDao.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/dao/StorePipelineRelDao.kt index 698245c0d56..3ac547fff81 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/dao/StorePipelineRelDao.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/dao/StorePipelineRelDao.kt @@ -114,4 +114,12 @@ class StorePipelineRelDao { .execute() } } + + fun deleteStorePipelineRelById(dslContext: DSLContext, id: String) { + with(TStorePipelineRel.T_STORE_PIPELINE_REL) { + dslContext.deleteFrom(this) + .where(ID.eq(id)) + .execute() + } + } } diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/dao/StoreProjectRelDao.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/dao/StoreProjectRelDao.kt index 52848bf401a..2ebc12cae21 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/dao/StoreProjectRelDao.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/dao/StoreProjectRelDao.kt @@ -31,6 +31,7 @@ import com.tencent.devops.common.api.util.UUIDUtil import com.tencent.devops.model.store.tables.TStoreMember import com.tencent.devops.model.store.tables.TStoreProjectRel import com.tencent.devops.model.store.tables.records.TStoreProjectRelRecord +import com.tencent.devops.store.pojo.common.StoreProjectInfo import com.tencent.devops.store.pojo.common.enums.StoreProjectTypeEnum import com.tencent.devops.store.pojo.common.enums.StoreTypeEnum import java.time.LocalDateTime @@ -265,7 +266,8 @@ class StoreProjectRelDao { storeProjectTypes?.let { conditions.add(TYPE.`in`(storeProjectTypes)) } - instanceId?.let { + // 测试中的应用对应的调试项目下无需判断测试版本的应用已安装 + if (!instanceId.isNullOrBlank() && storeProjectTypes?.contains(StoreProjectTypeEnum.TEST.type.toByte()) != true) { conditions.add(INSTANCE_ID.eq(instanceId)) } val baseQuery = dslContext.select(STORE_CODE, VERSION) @@ -398,6 +400,21 @@ class StoreProjectRelDao { return finalStep.fetchOne(0, String::class.java) } + /** + * 更新组件关联初始化项目信息 + */ + fun updateStoreInitProject(dslContext: DSLContext, userId: String, storeProjectInfo: StoreProjectInfo) { + with(TStoreProjectRel.T_STORE_PROJECT_REL) { + dslContext.update(this) + .set(PROJECT_CODE, storeProjectInfo.projectId) + .set(MODIFIER, userId) + .where(STORE_CODE.eq(storeProjectInfo.storeCode)) + .and(STORE_TYPE.eq(storeProjectInfo.storeType.type.toByte())) + .and(TYPE.eq(StoreProjectTypeEnum.INIT.type.toByte())) + .execute() + } + } + /** * 更新用户的组件设定的调试项目 */ @@ -558,4 +575,19 @@ class StoreProjectRelDao { .execute() } } + + fun getInitProjectInfoByStoreCode( + dslContext: DSLContext, + storeCode: String, + storeType: Byte + ): TStoreProjectRelRecord? { + with(TStoreProjectRel.T_STORE_PROJECT_REL) { + return dslContext.selectFrom(this) + .where(STORE_CODE.eq(storeCode) + .and(STORE_TYPE.eq(storeType)) + .and(TYPE.eq(StoreProjectTypeEnum.INIT.type.toByte())) + ) + .fetchOne() + } + } } diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/resources/OpStoreComponentResourceImpl.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/resources/OpStoreComponentResourceImpl.kt index 23cf4b2f0ed..3d9bb98dbd0 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/resources/OpStoreComponentResourceImpl.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/resources/OpStoreComponentResourceImpl.kt @@ -34,6 +34,7 @@ import com.tencent.devops.store.common.service.OpStoreComponentService import com.tencent.devops.store.common.service.StoreComponentManageService import com.tencent.devops.store.common.service.StoreComponentQueryService import com.tencent.devops.store.common.service.StoreReleaseService +import com.tencent.devops.store.pojo.common.InstalledPkgShaContentRequest import com.tencent.devops.store.pojo.common.MyStoreComponent import com.tencent.devops.store.pojo.common.QueryComponentsParam import com.tencent.devops.store.pojo.common.StoreBaseInfoUpdateRequest @@ -161,4 +162,20 @@ class OpStoreComponentResourceImpl @Autowired constructor( ) ) } + + override fun updateComponentInstalledPkgShaContent( + userId: String, + storeType: StoreTypeEnum, + storeCode: String, + version: String, + installedPkgShaContentRequest: InstalledPkgShaContentRequest + ): Result { + return storeComponentManageService.updateComponentInstalledPkgShaContent( + userId = userId, + storeType = storeType, + storeCode = storeCode, + version = version, + installedPkgShaContentRequest = installedPkgShaContentRequest + ) + } } diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/resources/OpStoreProjectResourceImpl.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/resources/OpStoreProjectResourceImpl.kt new file mode 100644 index 00000000000..958d0b6daac --- /dev/null +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/resources/OpStoreProjectResourceImpl.kt @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.store.common.resources + +import com.tencent.devops.common.api.pojo.Result +import com.tencent.devops.common.web.RestResource +import com.tencent.devops.store.api.common.OpStoreProjectResource +import com.tencent.devops.store.common.service.StoreProjectService +import com.tencent.devops.store.pojo.common.StoreProjectInfo +import org.springframework.beans.factory.annotation.Autowired + +@RestResource +class OpStoreProjectResourceImpl @Autowired constructor( + private val storeProjectService: StoreProjectService +) : OpStoreProjectResource { + override fun updateStoreInitProject(userId: String, storeProjectInfo: StoreProjectInfo): Result { + return Result(storeProjectService.updateStoreInitProject(userId, storeProjectInfo)) + } +} \ No newline at end of file diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/resources/OpenStoreResourceImpl.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/resources/OpenStoreResourceImpl.kt index d55da9b0439..0bdf189c548 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/resources/OpenStoreResourceImpl.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/resources/OpenStoreResourceImpl.kt @@ -33,6 +33,8 @@ import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.client.ClientTokenService import com.tencent.devops.common.redis.RedisOperation import com.tencent.devops.common.web.RestResource +import com.tencent.devops.common.web.annotation.BkApiPermission +import com.tencent.devops.common.web.constant.BkApiHandleType import com.tencent.devops.store.api.common.OpenStoreResource import com.tencent.devops.store.pojo.common.enums.StoreTypeEnum import com.tencent.devops.store.common.service.StoreCommonService @@ -48,6 +50,7 @@ class OpenStoreResourceImpl @Autowired constructor( private val redisOperation: RedisOperation ) : OpenStoreResource { + @BkApiPermission([BkApiHandleType.API_OPEN_TOKEN_CHECK]) override fun validateProjectComponentPermission( token: String, projectCode: String, diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/resources/ServiceSensitiveApiPermissionResourceImpl.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/resources/ServiceSensitiveApiPermissionResourceImpl.kt index 3cd92d461c0..6a96f522d58 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/resources/ServiceSensitiveApiPermissionResourceImpl.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/resources/ServiceSensitiveApiPermissionResourceImpl.kt @@ -30,8 +30,8 @@ package com.tencent.devops.store.common.resources import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.web.RestResource import com.tencent.devops.common.web.service.ServiceSensitiveApiPermissionResource -import com.tencent.devops.store.pojo.common.enums.StoreTypeEnum import com.tencent.devops.store.common.service.SensitiveApiService +import com.tencent.devops.store.pojo.common.enums.StoreTypeEnum import org.springframework.beans.factory.annotation.Autowired @RestResource @@ -39,11 +39,23 @@ class ServiceSensitiveApiPermissionResourceImpl @Autowired constructor( private val sensitiveApiService: SensitiveApiService ) : ServiceSensitiveApiPermissionResource { - override fun verifyApi(atomCode: String, apiName: String): Result { + override fun verifyApi( + installedPkgShaContent: String?, + osName: String?, + osArch: String?, + storeCode: String, + apiName: String, + storeType: String, + version: String? + ): Result { return sensitiveApiService.verifyApi( - storeType = StoreTypeEnum.ATOM, - storeCode = atomCode, - apiName = apiName + installedPkgShaContent = installedPkgShaContent, + osName = osName, + osArch = osArch, + storeType = StoreTypeEnum.valueOf(storeType), + storeCode = storeCode, + apiName = apiName, + version = version ) } } diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/resources/UserStoreMemberResourceImpl.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/resources/UserStoreMemberResourceImpl.kt index 406d4a8240d..44219e46698 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/resources/UserStoreMemberResourceImpl.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/resources/UserStoreMemberResourceImpl.kt @@ -73,8 +73,12 @@ class UserStoreMemberResourceImpl : UserStoreMemberResource { } private fun getStoreMemberService(storeType: StoreTypeEnum): StoreMemberService { - return SpringContextUtil.getBean( - StoreMemberService::class.java, - "${storeType.name.lowercase()}MemberService") + val beanName = "${storeType.name.lowercase()}MemberService" + return if (SpringContextUtil.isBeanExist(beanName)) { + SpringContextUtil.getBean(StoreMemberService::class.java, beanName) + } else { + // 获取默认的成员bean对象 + SpringContextUtil.getBean(StoreMemberService::class.java) + } } } diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/SensitiveApiService.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/SensitiveApiService.kt index 5dc15cd6182..f43149a80c0 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/SensitiveApiService.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/SensitiveApiService.kt @@ -70,9 +70,14 @@ interface SensitiveApiService { sensitiveApiApproveReq: SensitiveApiApproveReq ): Result + @Suppress("LongParameterList") fun verifyApi( + installedPkgShaContent: String? = null, + osName: String? = null, + osArch: String? = null, storeType: StoreTypeEnum, storeCode: String, - apiName: String + apiName: String, + version: String? = null ): Result } diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/StoreCommonService.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/StoreCommonService.kt index 173f94b0012..d667307c6e3 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/StoreCommonService.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/StoreCommonService.kt @@ -146,4 +146,12 @@ interface StoreCommonService { * 获取普通升级标识 */ fun getNormalUpgradeFlag(storeCode: String, storeType: StoreTypeEnum, status: StoreStatusEnum): Boolean + + /** + * 根据标识获取组件关联代码库hashId + */ + fun getStoreRepoHashIdByCode( + storeCode: String, + storeType: StoreTypeEnum + ): String? } diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/StoreComponentManageService.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/StoreComponentManageService.kt index d2b5cd418a4..c56e741db07 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/StoreComponentManageService.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/StoreComponentManageService.kt @@ -3,6 +3,7 @@ package com.tencent.devops.store.common.service import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.pipeline.enums.ChannelCode import com.tencent.devops.store.pojo.common.InstallStoreReq +import com.tencent.devops.store.pojo.common.InstalledPkgShaContentRequest import com.tencent.devops.store.pojo.common.StoreBaseInfoUpdateRequest import com.tencent.devops.store.pojo.common.UnInstallReq import com.tencent.devops.store.pojo.common.enums.StoreTypeEnum @@ -56,4 +57,15 @@ interface StoreComponentManageService { projectCode: String, userId: String ): Result + + /** + * 更新组件已安装包sha1摘要值 + */ + fun updateComponentInstalledPkgShaContent( + userId: String, + storeType: StoreTypeEnum, + storeCode: String, + version: String, + installedPkgShaContentRequest: InstalledPkgShaContentRequest + ): Result } diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/StoreProjectService.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/StoreProjectService.kt index e1c6333036c..3c7730e37c9 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/StoreProjectService.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/StoreProjectService.kt @@ -31,6 +31,7 @@ import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.pipeline.enums.ChannelCode import com.tencent.devops.store.pojo.common.InstallStoreReq import com.tencent.devops.store.pojo.common.InstalledProjRespItem +import com.tencent.devops.store.pojo.common.StoreProjectInfo import com.tencent.devops.store.pojo.common.enums.StoreTypeEnum /** @@ -102,4 +103,9 @@ interface StoreProjectService { storeProjectTypes: List, instanceId: String? = null ): Map? + + /** + * 更新组件初始化项目信息 + */ + fun updateStoreInitProject(userId: String, storeProjectInfo: StoreProjectInfo): Boolean } diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/impl/SensitiveApiServiceImpl.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/impl/SensitiveApiServiceImpl.kt index ffe4e7a15bf..56a87ab3a18 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/impl/SensitiveApiServiceImpl.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/impl/SensitiveApiServiceImpl.kt @@ -28,7 +28,10 @@ package com.tencent.devops.store.common.service.impl import com.fasterxml.jackson.core.type.TypeReference +import com.tencent.devops.common.api.auth.AUTH_HEADER_DEVOPS_SHA_CONTENT import com.tencent.devops.common.api.constant.CommonMessageCode +import com.tencent.devops.common.api.constant.KEY_INSTALLED_PKG_SHA_CONTENT +import com.tencent.devops.common.api.constant.KEY_VERSION import com.tencent.devops.common.api.exception.ErrorCodeException import com.tencent.devops.common.api.pojo.Page import com.tencent.devops.common.api.pojo.Result @@ -36,9 +39,16 @@ import com.tencent.devops.common.api.util.JsonUtil import com.tencent.devops.common.api.util.PageUtil import com.tencent.devops.common.api.util.PageUtil.DEFAULT_PAGE_SIZE import com.tencent.devops.common.api.util.UUIDUtil -import com.tencent.devops.store.constant.StoreMessageCode import com.tencent.devops.store.common.dao.BusinessConfigDao import com.tencent.devops.store.common.dao.SensitiveApiDao +import com.tencent.devops.store.common.dao.StoreBaseEnvExtQueryDao +import com.tencent.devops.store.common.dao.StoreBaseEnvQueryDao +import com.tencent.devops.store.common.dao.StoreBaseQueryDao +import com.tencent.devops.store.common.service.SensitiveApiService +import com.tencent.devops.store.constant.StoreMessageCode +import com.tencent.devops.store.pojo.common.enums.ApiLevelEnum +import com.tencent.devops.store.pojo.common.enums.ApiStatusEnum +import com.tencent.devops.store.pojo.common.enums.StoreTypeEnum import com.tencent.devops.store.pojo.common.sensitive.SensitiveApiApplyReq import com.tencent.devops.store.pojo.common.sensitive.SensitiveApiApproveReq import com.tencent.devops.store.pojo.common.sensitive.SensitiveApiConfig @@ -47,13 +57,7 @@ import com.tencent.devops.store.pojo.common.sensitive.SensitiveApiInfo import com.tencent.devops.store.pojo.common.sensitive.SensitiveApiNameInfo import com.tencent.devops.store.pojo.common.sensitive.SensitiveApiSearchDTO import com.tencent.devops.store.pojo.common.sensitive.SensitiveApiUpdateDTO -import com.tencent.devops.store.pojo.common.enums.ApiLevelEnum -import com.tencent.devops.store.pojo.common.enums.ApiStatusEnum -import com.tencent.devops.store.pojo.common.enums.BusinessEnum -import com.tencent.devops.store.pojo.common.enums.StoreTypeEnum -import com.tencent.devops.store.common.service.SensitiveApiService import org.jooq.DSLContext -import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service @@ -61,13 +65,15 @@ import org.springframework.stereotype.Service class SensitiveApiServiceImpl @Autowired constructor( private val dslContext: DSLContext, private val sensitiveApiDao: SensitiveApiDao, - private val businessConfigDao: BusinessConfigDao + private val businessConfigDao: BusinessConfigDao, + private val storeBaseQueryDao: StoreBaseQueryDao, + private val storeBaseEnvQueryDao: StoreBaseEnvQueryDao, + private val storeBaseEnvExtQueryDao: StoreBaseEnvExtQueryDao ) : SensitiveApiService { companion object { private const val BUSINESS_CONFIG_FEATURE = "api" private const val BUSINESS_CONFIG_VALUE = "sensitiveApi" - private val logger = LoggerFactory.getLogger(SensitiveApiServiceImpl::class.java) } override fun unApprovalApiList( @@ -76,7 +82,7 @@ class SensitiveApiServiceImpl @Autowired constructor( storeCode: String, language: String ): Result> { - val sensitiveApiConfigList = getSensitiveApiConfig() + val sensitiveApiConfigList = getSensitiveApiConfig(storeType) val approvedApiList = sensitiveApiDao.getApprovedApiNameList( dslContext = dslContext, storeCode = storeCode, @@ -89,10 +95,10 @@ class SensitiveApiServiceImpl @Autowired constructor( ) } - private fun getSensitiveApiConfig(): List { + private fun getSensitiveApiConfig(storeType: StoreTypeEnum): List { val businessConfigRecord = businessConfigDao.get( dslContext = dslContext, - business = BusinessEnum.ATOM.name, + business = storeType.name, feature = BUSINESS_CONFIG_FEATURE, businessValue = BUSINESS_CONFIG_VALUE ) ?: return emptyList() @@ -115,7 +121,7 @@ class SensitiveApiServiceImpl @Autowired constructor( params = arrayOf("applyDesc") ) } - val sensitiveApiNameMap = getSensitiveApiConfig().associateBy { it.apiName } + val sensitiveApiNameMap = getSensitiveApiConfig(storeType).associateBy { it.apiName } val sensitiveApiCreateDTOs = apiNameList.filter { it.isNotBlank() } .filter { sensitiveApiNameMap.containsKey(it) } @@ -223,10 +229,56 @@ class SensitiveApiServiceImpl @Autowired constructor( } override fun verifyApi( + installedPkgShaContent: String?, + osName: String?, + osArch: String?, storeType: StoreTypeEnum, storeCode: String, - apiName: String + apiName: String, + version: String? ): Result { + if (storeType == StoreTypeEnum.DEVX) { + if (version.isNullOrBlank()) { + throw ErrorCodeException(errorCode = CommonMessageCode.ERROR_NEED_PARAM_, params = arrayOf(KEY_VERSION)) + } + if (installedPkgShaContent.isNullOrBlank()) { + throw ErrorCodeException( + errorCode = CommonMessageCode.ERROR_NEED_PARAM_, + params = arrayOf(AUTH_HEADER_DEVOPS_SHA_CONTENT) + ) + } + // 判断请求中的sha1算法摘要值和db中的sha1算法摘要值是否能匹配 + val storeId = storeBaseQueryDao.getComponentId( + dslContext = dslContext, + storeCode = storeCode, + version = version, + storeType = storeType + ) ?: throw ErrorCodeException( + errorCode = CommonMessageCode.PARAMETER_IS_INVALID, + params = arrayOf("$storeType:$storeCode:$version") + ) + val baseEnvRecord = storeBaseEnvQueryDao.getBaseEnvsByStoreId( + dslContext = dslContext, + storeId = storeId, + osName = osName, + osArch = osArch + )?.getOrNull(0) ?: throw ErrorCodeException( + errorCode = CommonMessageCode.PARAMETER_IS_INVALID, + params = arrayOf("$osName:$osArch") + ) + val dbInstalledPkgShaContent = storeBaseEnvExtQueryDao.getBaseExtEnvsByEnvId( + dslContext = dslContext, + envId = baseEnvRecord.id, + fieldName = KEY_INSTALLED_PKG_SHA_CONTENT + )?.getOrNull(0)?.fieldValue ?: baseEnvRecord.shaContent + if (installedPkgShaContent.lowercase() != dbInstalledPkgShaContent) { + throw ErrorCodeException( + errorCode = CommonMessageCode.PARAMETER_VALIDATE_ERROR, + params = arrayOf(AUTH_HEADER_DEVOPS_SHA_CONTENT, "wrong sha1 content") + ) + } + } + // 判断组件是否有使用该API接口的权限 val record = sensitiveApiDao.getByApiName( dslContext = dslContext, storeType = storeType, diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/impl/StoreCommonServiceImpl.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/impl/StoreCommonServiceImpl.kt index 269423a252f..0f713da6198 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/impl/StoreCommonServiceImpl.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/impl/StoreCommonServiceImpl.kt @@ -203,6 +203,10 @@ abstract class StoreCommonServiceImpl : StoreCommonService { return publicFlag } + override fun getStoreRepoHashIdByCode(storeCode: String, storeType: StoreTypeEnum): String? { + return getStoreCommonDao(storeType.name).getStoreRepoHashIdByCode(dslContext, storeCode) + } + private fun getStoreCommonDao(storeType: String): AbstractStoreCommonDao { return SpringContextUtil.getBean(AbstractStoreCommonDao::class.java, "${storeType}_COMMON_DAO") } diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/impl/StoreComponentManageServiceImpl.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/impl/StoreComponentManageServiceImpl.kt index d7d55b89098..e66e9d80454 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/impl/StoreComponentManageServiceImpl.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/impl/StoreComponentManageServiceImpl.kt @@ -29,6 +29,7 @@ package com.tencent.devops.store.common.service.impl import com.tencent.devops.common.api.auth.AUTH_HEADER_USER_ID import com.tencent.devops.common.api.constant.CommonMessageCode +import com.tencent.devops.common.api.constant.KEY_INSTALLED_PKG_SHA_CONTENT import com.tencent.devops.common.api.exception.ErrorCodeException import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.api.util.UUIDUtil @@ -39,6 +40,8 @@ import com.tencent.devops.common.service.utils.SpringContextUtil import com.tencent.devops.model.store.tables.records.TStoreBaseRecord import com.tencent.devops.store.common.dao.ClassifyDao import com.tencent.devops.store.common.dao.ReasonRelDao +import com.tencent.devops.store.common.dao.StoreBaseEnvExtManageDao +import com.tencent.devops.store.common.dao.StoreBaseEnvQueryDao import com.tencent.devops.store.common.dao.StoreBaseExtManageDao import com.tencent.devops.store.common.dao.StoreBaseFeatureExtManageDao import com.tencent.devops.store.common.dao.StoreBaseFeatureManageDao @@ -59,11 +62,13 @@ import com.tencent.devops.store.common.utils.StoreReleaseUtils import com.tencent.devops.store.common.utils.StoreUtils import com.tencent.devops.store.constant.StoreMessageCode import com.tencent.devops.store.pojo.common.InstallStoreReq +import com.tencent.devops.store.pojo.common.InstalledPkgShaContentRequest import com.tencent.devops.store.pojo.common.StoreBaseInfoUpdateRequest import com.tencent.devops.store.pojo.common.UnInstallReq import com.tencent.devops.store.pojo.common.enums.ReasonTypeEnum import com.tencent.devops.store.pojo.common.enums.StoreStatusEnum import com.tencent.devops.store.pojo.common.enums.StoreTypeEnum +import com.tencent.devops.store.pojo.common.publication.StoreBaseEnvExtDataPO import com.tencent.devops.store.pojo.common.publication.StoreDeleteRequest import org.jooq.DSLContext import org.jooq.impl.DSL @@ -119,6 +124,12 @@ class StoreComponentManageServiceImpl : StoreComponentManageService { @Autowired lateinit var storeBaseFeatureQueryDao: StoreBaseFeatureQueryDao + @Autowired + lateinit var storeBaseEnvQueryDao: StoreBaseEnvQueryDao + + @Autowired + lateinit var storeBaseEnvExtManageDao: StoreBaseEnvExtManageDao + @Autowired lateinit var client: Client @@ -416,6 +427,38 @@ class StoreComponentManageServiceImpl : StoreComponentManageService { return Result(true) } + override fun updateComponentInstalledPkgShaContent( + userId: String, + storeType: StoreTypeEnum, + storeCode: String, + version: String, + installedPkgShaContentRequest: InstalledPkgShaContentRequest + ): Result { + val storeId = storeBaseQueryDao.getComponentId( + dslContext = dslContext, + storeCode = storeCode, + version = version, + storeType = storeType + ) ?: throw ErrorCodeException(errorCode = CommonMessageCode.ERROR_CLIENT_REST_ERROR) + val baseEnvRecord = storeBaseEnvQueryDao.getBaseEnvsByStoreId( + dslContext = dslContext, + storeId = storeId, + osName = installedPkgShaContentRequest.osName, + osArch = installedPkgShaContentRequest.osArch + )?.get(0) ?: throw ErrorCodeException(errorCode = CommonMessageCode.ERROR_CLIENT_REST_ERROR) + val storeBaseEnvExtDataPO = StoreBaseEnvExtDataPO( + id = UUIDUtil.generate(), + envId = baseEnvRecord.id, + storeId = storeId, + fieldName = KEY_INSTALLED_PKG_SHA_CONTENT, + fieldValue = installedPkgShaContentRequest.installedPkgShaContent, + creator = userId, + modifier = userId + ) + storeBaseEnvExtManageDao.batchSave(dslContext, listOf(storeBaseEnvExtDataPO)) + return Result(true) + } + private fun getStoreBaseInstallService(storeType: StoreTypeEnum): StoreBaseInstallService { val beanName = when (storeType) { StoreTypeEnum.TEMPLATE -> { diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/impl/StoreComponentMemberServiceImpl.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/impl/StoreComponentMemberServiceImpl.kt new file mode 100644 index 00000000000..5c31d4c74a9 --- /dev/null +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/impl/StoreComponentMemberServiceImpl.kt @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available. + * + * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved. + * + * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license. + * + * A copy of the MIT License is included in this file. + * + * + * Terms of the MIT License: + * --------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT + * LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package com.tencent.devops.store.common.service.impl + +import com.tencent.devops.store.common.dao.StoreBaseQueryDao +import com.tencent.devops.store.pojo.common.enums.StoreTypeEnum +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.context.annotation.Primary +import org.springframework.stereotype.Service + +@Primary +@Service +class StoreComponentMemberServiceImpl @Autowired constructor( + private val storeBaseQueryDao: StoreBaseQueryDao +) : StoreMemberServiceImpl() { + + override fun getStoreName(storeCode: String, storeType: StoreTypeEnum): String { + return storeBaseQueryDao.getLatestComponentByCode( + dslContext = dslContext, + storeCode = storeCode, + storeType = storeType + )?.name ?: "" + } +} diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/impl/StoreComponentQueryServiceImpl.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/impl/StoreComponentQueryServiceImpl.kt index 7da9c718d60..092d3aa6e55 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/impl/StoreComponentQueryServiceImpl.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/impl/StoreComponentQueryServiceImpl.kt @@ -769,7 +769,6 @@ class StoreComponentQueryServiceImpl : StoreComponentQueryService { ) { val projectCode = storeInfoQuery.projectCode!! val storeType = StoreTypeEnum.valueOf(storeInfoQuery.storeType) - val validStoreCodes = mutableSetOf() // 查询项目下已安装的组件版本信息 val installComponentMap = storeProjectService.getProjectComponents( projectCode = projectCode, @@ -779,7 +778,6 @@ class StoreComponentQueryServiceImpl : StoreComponentQueryService { ), instanceId = storeInfoQuery.instanceId ) ?: emptyMap() - validStoreCodes.addAll(installComponentMap.keys) // 查询项目下可调试的组件版本信息 val testComponentMap = storeProjectService.getProjectComponents( projectCode = projectCode, @@ -789,25 +787,25 @@ class StoreComponentQueryServiceImpl : StoreComponentQueryService { ), instanceId = storeInfoQuery.instanceId ) ?: emptyMap() - validStoreCodes.addAll(testComponentMap.keys) val publicComponentList = storeBaseFeatureQueryDao.getAllPublicComponent(dslContext, storeType) - validStoreCodes.addAll(publicComponentList) val tStoreBase = TStoreBase.T_STORE_BASE - // 查询非测试或者审核中组件最新发布版本信息 - val componentVersionMap = storeBaseQueryDao.getValidComponentsByCodes( - dslContext = dslContext, - storeCodes = validStoreCodes, - storeType = storeType, - testComponentFlag = false - ).intoMap({ it[tStoreBase.STORE_CODE] }, { it[tStoreBase.VERSION] }).toMutableMap() // 查询测试或者审核中组件最新版本信息 val testComponentVersionMap = storeBaseQueryDao.getValidComponentsByCodes( dslContext = dslContext, - storeCodes = validStoreCodes, + storeCodes = testComponentMap.keys.plus(publicComponentList), storeType = storeType, testComponentFlag = true ).intoMap({ it[tStoreBase.STORE_CODE] }, { it[tStoreBase.VERSION] }) val testStoreCodes = testComponentVersionMap.keys + // 查询非测试或者审核中组件最新发布版本信息 + val normalStoreCodes = installComponentMap.keys.plus(publicComponentList).toMutableSet() + normalStoreCodes.removeAll(testStoreCodes) + val componentVersionMap = storeBaseQueryDao.getValidComponentsByCodes( + dslContext = dslContext, + storeCodes = normalStoreCodes, + storeType = storeType, + testComponentFlag = false + ).intoMap({ it[tStoreBase.STORE_CODE] }, { it[tStoreBase.VERSION] }).toMutableMap() componentVersionMap.putAll(testComponentVersionMap) // 比较当前安装的版本与组件最新版本 val finalNormalStoreCodes = mutableSetOf() diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/impl/StoreMemberServiceImpl.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/impl/StoreMemberServiceImpl.kt index 55f8ad8a24b..179f3d631c8 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/impl/StoreMemberServiceImpl.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/impl/StoreMemberServiceImpl.kt @@ -246,7 +246,7 @@ abstract class StoreMemberServiceImpl : StoreMemberService { } if (sendNotify) { executorService.submit> { - val bodyParams = mapOf("storeAdmin" to userId, "storeName" to getStoreName(storeCode)) + val bodyParams = mapOf("storeAdmin" to userId, "storeName" to getStoreName(storeCode, storeType)) storeNotifyService.sendNotifyMessage( templateCode = STORE_MEMBER_ADD_NOTIFY_TEMPLATE + "_$storeType", sender = DEVOPS, @@ -304,7 +304,7 @@ abstract class StoreMemberServiceImpl : StoreMemberService { } executorService.submit> { val receivers = mutableSetOf(record.username) - val bodyParams = mapOf("storeAdmin" to userId, "storeName" to getStoreName(storeCode)) + val bodyParams = mapOf("storeAdmin" to userId, "storeName" to getStoreName(storeCode, storeType)) storeNotifyService.sendNotifyMessage( templateCode = STORE_MEMBER_DELETE_NOTIFY_TEMPLATE + "_$storeType", sender = DEVOPS, @@ -319,7 +319,7 @@ abstract class StoreMemberServiceImpl : StoreMemberService { /** * 获取组件名称 */ - abstract fun getStoreName(storeCode: String): String + abstract fun getStoreName(storeCode: String, storeType: StoreTypeEnum): String /** * 更改store组件成员的调试项目 diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/impl/StoreProjectServiceImpl.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/impl/StoreProjectServiceImpl.kt index e48c7b3c462..6eabb73ba41 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/impl/StoreProjectServiceImpl.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/common/service/impl/StoreProjectServiceImpl.kt @@ -30,6 +30,7 @@ package com.tencent.devops.store.common.service.impl import com.tencent.devops.common.api.constant.CommonMessageCode import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.api.util.DateTimeUtil +import com.tencent.devops.common.api.util.HashUtil import com.tencent.devops.common.api.util.Watcher import com.tencent.devops.common.client.Client import com.tencent.devops.common.pipeline.enums.ChannelCode @@ -37,15 +38,21 @@ import com.tencent.devops.common.redis.RedisLock import com.tencent.devops.common.redis.RedisOperation import com.tencent.devops.common.service.utils.LogUtils import com.tencent.devops.common.web.utils.I18nUtil +import com.tencent.devops.process.api.service.ServicePipelineResource import com.tencent.devops.project.api.service.ServiceProjectResource +import com.tencent.devops.repository.api.ServiceRepositoryResource +import com.tencent.devops.store.common.dao.StorePipelineBuildRelDao +import com.tencent.devops.store.common.dao.StorePipelineRelDao import com.tencent.devops.store.common.dao.StoreProjectRelDao import com.tencent.devops.store.common.dao.StoreStatisticDailyDao import com.tencent.devops.store.common.dao.StoreStatisticDao +import com.tencent.devops.store.common.service.StoreCommonService import com.tencent.devops.store.common.service.StoreProjectService import com.tencent.devops.store.common.service.StoreUserService import com.tencent.devops.store.constant.StoreMessageCode import com.tencent.devops.store.pojo.common.InstallStoreReq import com.tencent.devops.store.pojo.common.InstalledProjRespItem +import com.tencent.devops.store.pojo.common.StoreProjectInfo import com.tencent.devops.store.pojo.common.enums.StoreProjectTypeEnum import com.tencent.devops.store.pojo.common.enums.StoreTypeEnum import com.tencent.devops.store.pojo.common.statistic.StoreDailyStatisticRequest @@ -69,7 +76,10 @@ class StoreProjectServiceImpl @Autowired constructor( private val storeStatisticDao: StoreStatisticDao, private val storeStatisticDailyDao: StoreStatisticDailyDao, private val storeUserService: StoreUserService, - private val redisOperation: RedisOperation + private val redisOperation: RedisOperation, + private val storePipelineRelDao: StorePipelineRelDao, + private val storePipelineBuildRelDao: StorePipelineBuildRelDao, + private val storeCommonService: StoreCommonService ) : StoreProjectService { /** @@ -358,4 +368,59 @@ class StoreProjectServiceImpl @Autowired constructor( instanceId = instanceId ) } + + override fun updateStoreInitProject(userId: String, storeProjectInfo: StoreProjectInfo): Boolean { + dslContext.transaction { configuration -> + val context = DSL.using(configuration) + // 获取组件当前初始化项目 + val initProjectInfo = storeProjectRelDao.getInitProjectInfoByStoreCode( + dslContext = context, + storeCode = storeProjectInfo.storeCode, + storeType = storeProjectInfo.storeType.type.toByte() + )!! + // 更新组件关联初始化项目 + storeProjectRelDao.updateStoreInitProject(context, userId, storeProjectInfo) + storeProjectRelDao.deleteUserStoreTestProject( + dslContext = context, + userId = initProjectInfo.creator, + storeType = storeProjectInfo.storeType, + storeCode = storeProjectInfo.storeCode, + storeProjectType = StoreProjectTypeEnum.TEST + ) + storeProjectRelDao.addStoreProjectRel( + dslContext = context, + userId = userId, + storeType = storeProjectInfo.storeType.type.toByte(), + storeCode = storeProjectInfo.storeCode, + projectCode = storeProjectInfo.projectId, + type = StoreProjectTypeEnum.TEST.type.toByte() + ) + val storePipelineRel = storePipelineRelDao.getStorePipelineRel( + dslContext = context, + storeCode = storeProjectInfo.storeCode, + storeType = storeProjectInfo.storeType + ) + storePipelineRel?.let { + storePipelineRelDao.deleteStorePipelineRelById(context, storePipelineRel.id) + storePipelineBuildRelDao.deleteStorePipelineBuildRelByPipelineId(context, storePipelineRel.pipelineId) + client.get(ServicePipelineResource::class).delete( + userId = userId, + pipelineId = it.pipelineId, + channelCode = ChannelCode.AM, + projectId = initProjectInfo.projectCode, + checkFlag = false + ) + } + val storeRepoHashId = + storeCommonService.getStoreRepoHashIdByCode(storeProjectInfo.storeCode, storeProjectInfo.storeType) + storeRepoHashId?.let { + client.get(ServiceRepositoryResource::class).updateStoreRepoProject( + userId = storeProjectInfo.userId, + projectId = storeProjectInfo.projectId, + repositoryId = HashUtil.decodeOtherIdToLong(storeRepoHashId) + ) + } + } + return true + } } diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/image/dao/ImageCommonDao.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/image/dao/ImageCommonDao.kt index 3663314f9be..252e07d262b 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/image/dao/ImageCommonDao.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/image/dao/ImageCommonDao.kt @@ -155,4 +155,8 @@ class ImageCommonDao : AbstractStoreCommonDao() { null } } + + override fun getStoreRepoHashIdByCode(dslContext: DSLContext, storeCode: String): String? { + return null + } } diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/image/service/ImageMemberService.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/image/service/ImageMemberService.kt index f01de862adf..f8796b0a532 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/image/service/ImageMemberService.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/image/service/ImageMemberService.kt @@ -28,6 +28,7 @@ package com.tencent.devops.store.image.service import com.tencent.devops.store.common.service.impl.StoreMemberServiceImpl import com.tencent.devops.store.image.dao.MarketImageDao +import com.tencent.devops.store.pojo.common.enums.StoreTypeEnum import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service @@ -37,7 +38,7 @@ class ImageMemberService : StoreMemberServiceImpl() { @Autowired lateinit var marketImageDao: MarketImageDao - override fun getStoreName(storeCode: String): String { + override fun getStoreName(storeCode: String, storeType: StoreTypeEnum): String { return marketImageDao.getLatestImageByCode(dslContext, storeCode)?.imageName ?: "" } } diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/template/dao/TemplateCommonDao.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/template/dao/TemplateCommonDao.kt index 2f45eddae87..55ca642492b 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/template/dao/TemplateCommonDao.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/template/dao/TemplateCommonDao.kt @@ -68,6 +68,10 @@ class TemplateCommonDao : AbstractStoreCommonDao() { } } + override fun getStoreRepoHashIdByCode(dslContext: DSLContext, storeCode: String): String? { + return null + } + override fun getStorePublicFlagByCode(dslContext: DSLContext, storeCode: String): Boolean { return with(TTemplate.T_TEMPLATE) { dslContext.select(PUBLIC_FLAG).from(this) diff --git a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/template/service/impl/TemplateMemberServiceImpl.kt b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/template/service/impl/TemplateMemberServiceImpl.kt index 51a832ee51c..bbf2ccab350 100644 --- a/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/template/service/impl/TemplateMemberServiceImpl.kt +++ b/src/backend/ci/core/store/biz-store/src/main/kotlin/com/tencent/devops/store/template/service/impl/TemplateMemberServiceImpl.kt @@ -29,6 +29,7 @@ package com.tencent.devops.store.template.service.impl import com.tencent.devops.store.template.dao.MarketTemplateDao import com.tencent.devops.store.common.service.impl.StoreMemberServiceImpl +import com.tencent.devops.store.pojo.common.enums.StoreTypeEnum import org.springframework.beans.factory.annotation.Autowired import org.springframework.stereotype.Service @@ -38,7 +39,7 @@ class TemplateMemberServiceImpl : StoreMemberServiceImpl() { @Autowired private lateinit var marketTemplateDao: MarketTemplateDao - override fun getStoreName(storeCode: String): String { + override fun getStoreName(storeCode: String, storeType: StoreTypeEnum): String { return marketTemplateDao.getLatestTemplateByCode(dslContext, storeCode)?.templateName ?: "" } } diff --git a/src/backend/ci/core/worker/worker-agent/src/main/resources/.agent.properties b/src/backend/ci/core/worker/worker-agent/src/main/resources/.agent.properties index 24343bffb64..7e156ac392d 100644 --- a/src/backend/ci/core/worker/worker-agent/src/main/resources/.agent.properties +++ b/src/backend/ci/core/worker/worker-agent/src/main/resources/.agent.properties @@ -32,6 +32,5 @@ landun.gateway=##gateWay## devops.parallel.task.count=4 landun.env=##landun.env## agentCollectorOn=##agentCollectorOn## -repo.class.name=com.tencent.devops.agent.service.SampleRepoServiceImpl devops.public.host.maxFileCacheSize=209715200 devops.public.third.maxFileCacheSize=2147483648 diff --git a/src/backend/ci/core/worker/worker-agent/src/test/kotlin/com/tencent/devops/worker/common/utils/CommandLineUtilsTest.kt b/src/backend/ci/core/worker/worker-agent/src/test/kotlin/com/tencent/devops/worker/common/utils/CommandLineUtilsTest.kt new file mode 100644 index 00000000000..8ca9f96a843 --- /dev/null +++ b/src/backend/ci/core/worker/worker-agent/src/test/kotlin/com/tencent/devops/worker/common/utils/CommandLineUtilsTest.kt @@ -0,0 +1,127 @@ +package com.tencent.devops.worker.common.utils + +import java.io.File +import java.security.AccessController +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test +import sun.security.action.GetPropertyAction + +class CommandLineUtilsTest { + + @Test + fun reportProgressRateTest() { + fun func(str: String) = CommandLineUtils.reportProgressRate("test", str) + /*不识别*/ + Assertions.assertEquals(func("echo \"::set-progress-rate 0.3758\""), null) + Assertions.assertEquals(func("echo '::set-progress-rate 0.3758'"), null) + Assertions.assertEquals(func("echo ::set-progress-rate 0.3758"), null) + Assertions.assertEquals(func("print(\"::set-progress-rate 0.3758\")"), null) + /*windows兼容*/ + Assertions.assertEquals(func("\"::set-progress-rate 0.3758\""), 0.3758) + /*默认*/ + Assertions.assertEquals(func("::set-progress-rate 0.3758"), 0.3758) + /*兼容多空格*/ + Assertions.assertEquals(func("::set-progress-rate 0.3758"), 0.3758) + Assertions.assertEquals(func(" ::set-progress-rate 0.3758"), 0.3758) + } + + @Test + fun appendVariableToFileTest() { + fun func(str: String) = CommandLineUtils.appendVariableToFile( + str, File( + AccessController + .doPrivileged(GetPropertyAction("java.io.tmpdir")) + ), "appendVariableToFileTest" + ) + /*不识别*/ + Assertions.assertEquals(func("echo \"::set-variable name=RESULT::test\""), null) + Assertions.assertEquals(func("echo '::set-variable name=RESULT::test'"), null) + Assertions.assertEquals(func("echo ::set-variable name=RESULT::test"), null) + Assertions.assertEquals(func("print(\"::set-variable name=RESULT::test\")"), null) + /*多空格*/ + Assertions.assertEquals(func(" ::set-variable name=RESULT::test"), null) + Assertions.assertEquals(func("::set-variable name=RESULT::test"), null) + /*windows兼容*/ + Assertions.assertEquals(func("\"::set-variable name=RESULT::test\""), "variables.RESULT=test\n") + /*默认*/ + Assertions.assertEquals(func("::set-variable name=RESULT::test"), "variables.RESULT=test\n") + } + + @Test + fun appendRemarkToFileTest() { + fun func(str: String) = CommandLineUtils.appendRemarkToFile( + str, File( + AccessController + .doPrivileged(GetPropertyAction("java.io.tmpdir")) + ), "appendRemarkToFileTest" + ) + /*不识别*/ + Assertions.assertEquals(func("echo \"::set-remark 备注信息\""), null) + Assertions.assertEquals(func("echo '::set-remark 备注信息'"), null) + Assertions.assertEquals(func("echo ::set-remark 备注信息"), null) + Assertions.assertEquals(func("print(\"::set-remark 备注信息\")"), null) + /*多空格*/ + Assertions.assertEquals(func(" ::set-remark 备注信息"), null) + /*windows兼容*/ + Assertions.assertEquals(func("\"::set-remark 备注信息\""), "BK_CI_BUILD_REMARK=备注信息\n") + /*默认*/ + Assertions.assertEquals(func("::set-remark 备注信息"), "BK_CI_BUILD_REMARK=备注信息\n") + Assertions.assertEquals(func("::set-remark 备注信息"), "BK_CI_BUILD_REMARK= 备注信息\n") + } + + @Test + fun appendOutputToFileTest() { + val jobId = "job_xx" + val stepId = "step_xx" + fun func(str: String) = CommandLineUtils.appendOutputToFile( + tmpLine = str, + workspace = File( + AccessController + .doPrivileged(GetPropertyAction("java.io.tmpdir")) + ), + resultLogFile = "appendOutputToFileTest", + jobId = jobId, + stepId = stepId + ) + /*不识别*/ + Assertions.assertEquals(func("echo \"::set-output name=RESULT::test\""), null) + Assertions.assertEquals(func("echo '::set-output name=RESULT::test'"), null) + Assertions.assertEquals(func("echo ::set-output name=RESULT::test"), null) + Assertions.assertEquals(func("print(\"::set-output name=RESULT::test\")"), null) + /*多空格*/ + Assertions.assertEquals(func(" ::set-output name=RESULT::test"), null) + Assertions.assertEquals(func("::set-output name=RESULT::test"), null) + /*windows兼容*/ + Assertions.assertEquals( + func("\"::set-output name=RESULT::test\""), + "jobs.$jobId.steps.$stepId.outputs.RESULT=test\n" + ) + /*默认*/ + Assertions.assertEquals( + func("::set-output name=RESULT::test"), + "jobs.$jobId.steps.$stepId.outputs.RESULT=test\n" + ) + } + + @Test + fun appendGateToFileTest() { + fun func(str: String) = CommandLineUtils.appendGateToFile( + str, File( + AccessController + .doPrivileged(GetPropertyAction("java.io.tmpdir")) + ), "appendGateToFileTest" + ) + /*不识别*/ + Assertions.assertEquals(func("echo \"::set-gate-value name=pass_rate::0.9\""), null) + Assertions.assertEquals(func("echo '::set-gate-value name=pass_rate::0.9'"), null) + Assertions.assertEquals(func("echo ::set-gate-value name=pass_rate::0.9"), null) + Assertions.assertEquals(func("print(\"::set-gate-value name=pass_rate::0.9\")"), null) + /*多空格*/ + Assertions.assertEquals(func(" ::set-gate-value name=pass_rate::0.9"), null) + Assertions.assertEquals(func("::set-gate-value name=pass_rate::0.9"), null) + /*windows兼容*/ + Assertions.assertEquals(func("\"::set-gate-value name=pass_rate::0.9\""), "pass_rate=0.9\n") + /*默认*/ + Assertions.assertEquals(func("::set-gate-value name=pass_rate::0.9"), "pass_rate=0.9\n") + } +} diff --git a/src/backend/ci/core/worker/worker-api-sdk/src/main/kotlin/com/tencent/devops/worker/common/api/archive/ArchiveResourceApi.kt b/src/backend/ci/core/worker/worker-api-sdk/src/main/kotlin/com/tencent/devops/worker/common/api/archive/ArchiveResourceApi.kt index e43e22d865b..1d7320adb55 100644 --- a/src/backend/ci/core/worker/worker-api-sdk/src/main/kotlin/com/tencent/devops/worker/common/api/archive/ArchiveResourceApi.kt +++ b/src/backend/ci/core/worker/worker-api-sdk/src/main/kotlin/com/tencent/devops/worker/common/api/archive/ArchiveResourceApi.kt @@ -29,6 +29,7 @@ package com.tencent.devops.worker.common.api.archive import com.fasterxml.jackson.module.kotlin.readValue import com.google.gson.JsonParser +import com.tencent.bkrepo.repository.pojo.token.TokenType import com.tencent.devops.artifactory.constant.REALM_LOCAL import com.tencent.devops.artifactory.pojo.GetFileDownloadUrlsResponse import com.tencent.devops.artifactory.pojo.enums.FileTypeEnum @@ -207,4 +208,15 @@ class ArchiveResourceApi : AbstractBuildResourceApi(), ArchiveSDKApi { val responseContent = request(request, "upload file:$fileName fail") return objectMapper.readValue(responseContent) } + + override fun getRepoToken( + userId: String, + projectId: String, + repoName: String, + path: String, + type: TokenType, + expireSeconds: Long + ): String? { + return null + } } diff --git a/src/backend/ci/core/worker/worker-api-sdk/src/main/kotlin/com/tencent/devops/worker/common/api/archive/BkRepoArchiveResourceApi.kt b/src/backend/ci/core/worker/worker-api-sdk/src/main/kotlin/com/tencent/devops/worker/common/api/archive/BkRepoArchiveResourceApi.kt index 4913a680f9a..6f8b3728618 100644 --- a/src/backend/ci/core/worker/worker-api-sdk/src/main/kotlin/com/tencent/devops/worker/common/api/archive/BkRepoArchiveResourceApi.kt +++ b/src/backend/ci/core/worker/worker-api-sdk/src/main/kotlin/com/tencent/devops/worker/common/api/archive/BkRepoArchiveResourceApi.kt @@ -29,6 +29,7 @@ package com.tencent.devops.worker.common.api.archive import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper import com.fasterxml.jackson.module.kotlin.readValue +import com.tencent.bkrepo.repository.pojo.token.TokenType import com.tencent.devops.artifactory.constant.REALM_BK_REPO import com.tencent.devops.artifactory.pojo.enums.FileTypeEnum import com.tencent.devops.common.api.exception.RemoteServiceException @@ -37,6 +38,7 @@ import com.tencent.devops.common.api.pojo.ErrorCode import com.tencent.devops.common.api.pojo.ErrorType import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.api.util.MessageUtil +import com.tencent.devops.common.util.HttpRetryUtils import com.tencent.devops.process.pojo.BuildVariables import com.tencent.devops.worker.common.api.AbstractBuildResourceApi import com.tencent.devops.worker.common.api.ApiPriority @@ -308,4 +310,28 @@ class BkRepoArchiveResourceApi : AbstractBuildResourceApi(), ArchiveSDKApi { val responseContent = request(request, "upload file[$fileName] failed") return objectMapper.readValue(responseContent) } + + override fun getRepoToken( + userId: String, + projectId: String, + repoName: String, + path: String, + type: TokenType, + expireSeconds: Long + ): String? { + return if (bkrepoResourceApi.tokenAccess()) { + HttpRetryUtils.retry(retryTime = 3, retryPeriodMills = 1000) { + bkrepoResourceApi.createBkRepoTemporaryToken( + userId = userId, + projectId = projectId, + repoName = repoName, + path = path, + type = type, + expireSeconds = expireSeconds + ) + } + } else { + null + } + } } diff --git a/src/backend/ci/core/worker/worker-api-sdk/src/main/kotlin/com/tencent/devops/worker/common/api/process/BuildResourceApi.kt b/src/backend/ci/core/worker/worker-api-sdk/src/main/kotlin/com/tencent/devops/worker/common/api/process/BuildResourceApi.kt index 1c927835de1..6f19010ad4f 100644 --- a/src/backend/ci/core/worker/worker-api-sdk/src/main/kotlin/com/tencent/devops/worker/common/api/process/BuildResourceApi.kt +++ b/src/backend/ci/core/worker/worker-api-sdk/src/main/kotlin/com/tencent/devops/worker/common/api/process/BuildResourceApi.kt @@ -144,10 +144,11 @@ class BuildResourceApi : AbstractBuildResourceApi(), BuildSDKApi { projectId: String, pipelineId: String, buildNum: String, - channelCode: ChannelCode? + channelCode: ChannelCode?, + buildId: String ): Result { - val sb = StringBuilder("/ms/process/api/build/builds/$projectId/$pipelineId/$buildNum/history") - if (channelCode != null) sb.append("?channelCode=${channelCode.name}") + val sb = StringBuilder("/ms/process/api/build/builds/$projectId/$pipelineId/$buildNum/history?buildId=$buildId") + if (channelCode != null) sb.append("&channelCode=${channelCode.name}") val path = sb.toString() val request = buildGet(path) val errorMessage = MessageUtil.getMessageByLocale( diff --git a/src/backend/ci/core/worker/worker-api-sdk/src/main/kotlin/com/tencent/devops/worker/common/api/report/BkRepoReportResourceApi.kt b/src/backend/ci/core/worker/worker-api-sdk/src/main/kotlin/com/tencent/devops/worker/common/api/report/BkRepoReportResourceApi.kt index d63bcc28774..4e794f8c5bf 100644 --- a/src/backend/ci/core/worker/worker-api-sdk/src/main/kotlin/com/tencent/devops/worker/common/api/report/BkRepoReportResourceApi.kt +++ b/src/backend/ci/core/worker/worker-api-sdk/src/main/kotlin/com/tencent/devops/worker/common/api/report/BkRepoReportResourceApi.kt @@ -28,10 +28,12 @@ package com.tencent.devops.worker.common.api.report import com.fasterxml.jackson.module.kotlin.readValue +import com.tencent.bkrepo.repository.pojo.token.TokenType import com.tencent.devops.artifactory.constant.REALM_BK_REPO import com.tencent.devops.common.api.exception.RemoteServiceException import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.common.api.util.MessageUtil +import com.tencent.devops.common.util.HttpRetryUtils import com.tencent.devops.process.pojo.BuildVariables import com.tencent.devops.process.pojo.report.ReportEmail import com.tencent.devops.process.pojo.report.enums.ReportTypeEnum @@ -197,4 +199,28 @@ class BkRepoReportResourceApi : AbstractBuildResourceApi(), ReportSDKApi { } bkrepoResourceApi.setPipelineMetadata("report", buildVariables) } + + override fun getRepoToken( + userId: String, + projectId: String, + repoName: String, + path: String, + type: TokenType, + expireSeconds: Long + ): String? { + return if (bkrepoResourceApi.tokenAccess()) { + HttpRetryUtils.retry(retryTime = 3, retryPeriodMills = 1000) { + bkrepoResourceApi.createBkRepoTemporaryToken( + userId = userId, + projectId = projectId, + repoName = repoName, + path = path, + type = type, + expireSeconds = expireSeconds + ) + } + } else { + null + } + } } diff --git a/src/backend/ci/core/worker/worker-api-sdk/src/main/kotlin/com/tencent/devops/worker/common/api/report/ReportResourceApi.kt b/src/backend/ci/core/worker/worker-api-sdk/src/main/kotlin/com/tencent/devops/worker/common/api/report/ReportResourceApi.kt index e630cfc4967..eed76bb621a 100644 --- a/src/backend/ci/core/worker/worker-api-sdk/src/main/kotlin/com/tencent/devops/worker/common/api/report/ReportResourceApi.kt +++ b/src/backend/ci/core/worker/worker-api-sdk/src/main/kotlin/com/tencent/devops/worker/common/api/report/ReportResourceApi.kt @@ -29,6 +29,7 @@ package com.tencent.devops.worker.common.api.report import com.fasterxml.jackson.module.kotlin.readValue import com.google.gson.JsonParser +import com.tencent.bkrepo.repository.pojo.token.TokenType import com.tencent.devops.artifactory.constant.REALM_LOCAL import com.tencent.devops.artifactory.pojo.enums.FileTypeEnum import com.tencent.devops.common.api.exception.RemoteServiceException @@ -134,6 +135,17 @@ class ReportResourceApi : AbstractBuildResourceApi(), ReportSDKApi { return objectMapper.readValue(responseContent) } + override fun getRepoToken( + userId: String, + projectId: String, + repoName: String, + path: String, + type: TokenType, + expireSeconds: Long + ): String? { + return null + } + companion object { private val logger = LoggerFactory.getLogger(ReportResourceApi::class.java) } diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/plugin/worker/task/archive/BuildArchiveGetTask.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/plugin/worker/task/archive/BuildArchiveGetTask.kt index 21789e2b2c2..f64b2b7090a 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/plugin/worker/task/archive/BuildArchiveGetTask.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/plugin/worker/task/archive/BuildArchiveGetTask.kt @@ -44,7 +44,6 @@ import com.tencent.devops.worker.common.api.ArtifactApiFactory import com.tencent.devops.worker.common.api.archive.ArchiveSDKApi import com.tencent.devops.worker.common.api.process.BuildSDKApi import com.tencent.devops.worker.common.logger.LoggerService -import com.tencent.devops.worker.common.service.RepoServiceFactory import com.tencent.devops.worker.common.task.ITask import com.tencent.devops.worker.common.task.TaskClassType import com.tencent.devops.worker.common.utils.TaskUtil @@ -68,7 +67,11 @@ class BuildArchiveGetTask : ITask() { val taskParams = buildTask.params ?: mapOf() val pipelineId = taskParams["pipelineId"] ?: throw ParamBlankException("pipelineId is null") val buildNo = if (taskParams["buildNo"].isNullOrBlank()) "-1" else taskParams["buildNo"]!! - val buildId = getBuildId(pipelineId, buildVariables, buildNo).id + val buildId = if (pipelineId == buildVariables.pipelineId && buildNo == "-1") { + buildVariables.buildId + } else { + getBuildId(pipelineId, buildVariables, buildNo).id + } val destPath = File(workspace, taskParams["destPath"] ?: ".") val srcPaths = taskParams["srcPaths"] ?: throw ParamBlankException("srcPaths can not be null") val notFoundContinue = taskParams["notFoundContinue"] ?: "false" @@ -104,7 +107,7 @@ class BuildArchiveGetTask : ITask() { count = files.size LoggerService.addNormalLine("total $count file(s) found") files.forEachIndexed { index, (fileUrl, file) -> - val token = RepoServiceFactory.getInstance().getRepoToken( + val token = archiveGetResourceApi.getRepoToken( userId = buildVariables.variables[PIPELINE_START_USER_ID] ?: "", projectId = buildVariables.projectId, repoName = "pipeline", @@ -136,7 +139,8 @@ class BuildArchiveGetTask : ITask() { projectId = buildVariables.projectId, pipelineId = pipelineId, buildNum = buildNo, - channelCode = ChannelCode.BS + channelCode = ChannelCode.BS, + buildId = buildVariables.buildId ).data ?: throw TaskExecuteException( errorCode = ErrorCode.USER_RESOURCE_NOT_FOUND, errorType = ErrorType.USER, diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/plugin/worker/task/archive/CustomizeArchiveGetTask.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/plugin/worker/task/archive/CustomizeArchiveGetTask.kt index a6bb7053d0c..79c14fc491d 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/plugin/worker/task/archive/CustomizeArchiveGetTask.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/plugin/worker/task/archive/CustomizeArchiveGetTask.kt @@ -39,7 +39,6 @@ import com.tencent.devops.process.utils.PIPELINE_START_USER_ID import com.tencent.devops.worker.common.api.ArtifactApiFactory import com.tencent.devops.worker.common.api.archive.ArchiveSDKApi import com.tencent.devops.worker.common.logger.LoggerService -import com.tencent.devops.worker.common.service.RepoServiceFactory import com.tencent.devops.worker.common.task.ITask import com.tencent.devops.worker.common.task.TaskClassType import com.tencent.devops.worker.common.utils.TaskUtil @@ -92,7 +91,7 @@ class CustomizeArchiveGetTask : ITask() { count = files.size LoggerService.addNormalLine("total $count file(s) found") files.forEachIndexed { index, (fileUrl, file) -> - val token = RepoServiceFactory.getInstance().getRepoToken( + val token = archiveGetResourceApi.getRepoToken( userId = buildVariables.variables[PIPELINE_START_USER_ID] ?: "", projectId = buildVariables.projectId, repoName = "custom", diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/plugin/worker/task/archive/ReportArchiveTask.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/plugin/worker/task/archive/ReportArchiveTask.kt index 8ca1d186a5c..591c304d499 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/plugin/worker/task/archive/ReportArchiveTask.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/plugin/worker/task/archive/ReportArchiveTask.kt @@ -50,10 +50,10 @@ import com.tencent.devops.worker.common.constants.WorkerMessageCode.FOLDER_NOT_E import com.tencent.devops.worker.common.constants.WorkerMessageCode.UPLOAD_CUSTOM_OUTPUT_SUCCESS import com.tencent.devops.worker.common.env.AgentEnv import com.tencent.devops.worker.common.logger.LoggerService -import com.tencent.devops.worker.common.service.RepoServiceFactory import com.tencent.devops.worker.common.task.ITask import com.tencent.devops.worker.common.task.TaskClassType import com.tencent.devops.worker.common.utils.TaskUtil +import org.slf4j.LoggerFactory import java.io.File import java.nio.file.Path import java.nio.file.Paths @@ -61,7 +61,6 @@ import java.util.concurrent.Executors import java.util.concurrent.TimeUnit import java.util.concurrent.TimeoutException import java.util.regex.Pattern -import org.slf4j.LoggerFactory @TaskClassType(classTypes = [ReportArchiveElement.classType]) class ReportArchiveTask : ITask() { @@ -81,7 +80,7 @@ class ReportArchiveTask : ITask() { val reportType = taskParams["reportType"] ?: ReportTypeEnum.INTERNAL.name val indexFileParam: String var indexFileContent: String - val token = RepoServiceFactory.getInstance().getRepoToken( + val token = api.getRepoToken( userId = buildVariables.variables[PIPELINE_START_USER_ID] ?: "", projectId = buildVariables.projectId, repoName = "report", diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/plugin/worker/task/archive/SingleFileArchiveTask.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/plugin/worker/task/archive/SingleFileArchiveTask.kt index e222f0e457c..6b2b6eda434 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/plugin/worker/task/archive/SingleFileArchiveTask.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/plugin/worker/task/archive/SingleFileArchiveTask.kt @@ -37,9 +37,10 @@ import com.tencent.devops.common.archive.element.SingleArchiveElement import com.tencent.devops.process.pojo.BuildTask import com.tencent.devops.process.pojo.BuildVariables import com.tencent.devops.process.utils.PIPELINE_START_USER_ID +import com.tencent.devops.worker.common.api.ArtifactApiFactory +import com.tencent.devops.worker.common.api.archive.ArchiveSDKApi import com.tencent.devops.worker.common.constants.WorkerMessageCode.NO_MATCHING_ARCHIVE_FILE import com.tencent.devops.worker.common.env.AgentEnv -import com.tencent.devops.worker.common.service.RepoServiceFactory import com.tencent.devops.worker.common.task.ITask import com.tencent.devops.worker.common.task.TaskClassType import com.tencent.devops.worker.common.utils.ArchiveUtils.archiveCustomFiles @@ -50,12 +51,14 @@ import java.io.File @TaskClassType(classTypes = [SingleArchiveElement.classType]) class SingleFileArchiveTask : ITask() { + private val api = ArtifactApiFactory.create(ArchiveSDKApi::class) + override fun execute(buildTask: BuildTask, buildVariables: BuildVariables, workspace: File) { val taskParams = buildTask.params ?: mapOf() val filePath = taskParams["filePath"] ?: throw ParamBlankException("param [filePath] is empty") val isCustomize = taskParams["customize"] ?: throw ParamBlankException("param [isCustomize] is empty") TaskUtil.setTaskId(buildTask.taskId ?: "") - val token = RepoServiceFactory.getInstance().getRepoToken( + val token = api.getRepoToken( userId = buildVariables.variables[PIPELINE_START_USER_ID] ?: "", projectId = buildVariables.projectId, repoName = if (isCustomize.toBoolean()) "custom" else "pipeline", diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/Runner.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/Runner.kt index 9ec61d2dce9..31be9ef9fe7 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/Runner.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/Runner.kt @@ -414,13 +414,15 @@ object Runner { ) { // 如果之前的插件不是在构建机执行, 会缺少环境变量 val taskBuildVariable = buildTask.buildVariable?.toMutableMap() ?: mutableMapOf() + val variablesWithType = jobBuildVariables.variablesWithType + .associateBy { it.key } + // job 变量能取到真实readonly，保证task 变量readOnly属性不会改变 val taskBuildParameters = taskBuildVariable.map { (key, value) -> - BuildParameters(key, value) + BuildParameters(key = key, value = value, readOnly = variablesWithType[key]?.readOnly) }.associateBy { it.key } jobBuildVariables.variables = jobBuildVariables.variables.plus(taskBuildVariable) // 以key去重, 并以buildTask中的为准 - jobBuildVariables.variablesWithType = jobBuildVariables.variablesWithType - .associateBy { it.key } + jobBuildVariables.variablesWithType = variablesWithType .plus(taskBuildParameters) .values.toList() diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/api/archive/ArchiveSDKApi.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/api/archive/ArchiveSDKApi.kt index e97a82dc0ab..7c1cf620f5e 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/api/archive/ArchiveSDKApi.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/api/archive/ArchiveSDKApi.kt @@ -27,6 +27,7 @@ package com.tencent.devops.worker.common.api.archive +import com.tencent.bkrepo.repository.pojo.token.TokenType import com.tencent.devops.artifactory.pojo.enums.FileTypeEnum import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.process.pojo.BuildVariables @@ -134,4 +135,16 @@ interface ArchiveSDKApi : WorkerRestApiSDK { headers: Map? = emptyMap(), isVmBuildEnv: Boolean? = null ): Result + + /** + * 获取仓库token + */ + fun getRepoToken( + userId: String, + projectId: String, + repoName: String, + path: String, + type: TokenType, + expireSeconds: Long + ): String? } diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/api/process/BuildSDKApi.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/api/process/BuildSDKApi.kt index e6c75046e4c..83407dce344 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/api/process/BuildSDKApi.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/api/process/BuildSDKApi.kt @@ -47,7 +47,8 @@ interface BuildSDKApi : WorkerRestApiSDK { projectId: String, pipelineId: String, buildNum: String, - channelCode: ChannelCode? + channelCode: ChannelCode?, + buildId: String ): Result fun getBuildDetail( diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/api/report/ReportSDKApi.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/api/report/ReportSDKApi.kt index be461a6db16..db63f1e4bc6 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/api/report/ReportSDKApi.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/api/report/ReportSDKApi.kt @@ -27,6 +27,7 @@ package com.tencent.devops.worker.common.api.report +import com.tencent.bkrepo.repository.pojo.token.TokenType import com.tencent.devops.common.api.pojo.Result import com.tencent.devops.process.pojo.BuildVariables import com.tencent.devops.process.pojo.report.ReportEmail @@ -75,4 +76,16 @@ interface ReportSDKApi : WorkerRestApiSDK { buildVariables: BuildVariables, token: String? = null ) + + /** + * 获取仓库token + */ + fun getRepoToken( + userId: String, + projectId: String, + repoName: String, + path: String, + type: TokenType, + expireSeconds: Long + ): String? } diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/logger/LoggerService.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/logger/LoggerService.kt index 0a20c75b8ea..7fcad41c9c4 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/logger/LoggerService.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/logger/LoggerService.kt @@ -46,9 +46,9 @@ import com.tencent.devops.worker.common.LOG_SUBTAG_FLAG import com.tencent.devops.worker.common.LOG_TASK_LINE_LIMIT import com.tencent.devops.worker.common.LOG_WARN_FLAG import com.tencent.devops.worker.common.api.ApiFactory +import com.tencent.devops.worker.common.api.archive.ArchiveSDKApi import com.tencent.devops.worker.common.api.log.LogSDKApi import com.tencent.devops.worker.common.env.AgentEnv -import com.tencent.devops.worker.common.service.RepoServiceFactory import com.tencent.devops.worker.common.service.SensitiveValueService import com.tencent.devops.worker.common.utils.ArchiveUtils import com.tencent.devops.worker.common.utils.FileUtils @@ -72,6 +72,7 @@ import java.util.concurrent.locks.ReentrantLock object LoggerService { private val logResourceApi = ApiFactory.create(LogSDKApi::class) + private val archiveApi = ApiFactory.create(ArchiveSDKApi::class) private val logger = LoggerFactory.getLogger(LoggerService::class.java) private var future: Future? = null private val running = AtomicBoolean(true) @@ -365,7 +366,7 @@ object LoggerService { logger.info("Start to archive log files with LogMode[${AgentEnv.getLogMode()}]") try { val expireSeconds = buildVariables!!.timeoutMills / 1000 - val token = RepoServiceFactory.getInstance().getRepoToken( + val token = archiveApi.getRepoToken( userId = buildVariables!!.variables[PIPELINE_START_USER_ID] ?: "", projectId = buildVariables!!.projectId, repoName = "log", diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/ITask.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/ITask.kt index 4030a6e7dac..90b707e2e39 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/ITask.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/ITask.kt @@ -27,13 +27,18 @@ package com.tencent.devops.worker.common.task +import com.tencent.devops.common.api.exception.TaskExecuteException +import com.tencent.devops.common.api.pojo.ErrorCode +import com.tencent.devops.common.api.pojo.ErrorType import com.tencent.devops.common.pipeline.pojo.BuildParameters import com.tencent.devops.process.pojo.BuildTask import com.tencent.devops.process.pojo.BuildVariables import com.tencent.devops.worker.common.env.BuildEnv import com.tencent.devops.worker.common.env.BuildType -import org.slf4j.LoggerFactory +import com.tencent.devops.worker.common.logger.LoggerService import java.io.File +import java.util.stream.Collectors +import org.slf4j.LoggerFactory @Suppress("NestedBlockDepth", "TooManyFunctions") abstract class ITask { @@ -50,11 +55,18 @@ abstract class ITask { private var finishKillFlag: Boolean? = null + /* 存储常量的key */ + private lateinit var constVar: List + fun run( buildTask: BuildTask, buildVariables: BuildVariables, workspace: File ) { + constVar = buildVariables.variablesWithType.stream() + .filter { it.readOnly == true } + .map { it.key } + .collect(Collectors.toList()) execute(buildTask, buildVariables, workspace) } @@ -83,6 +95,23 @@ abstract class ITask { ) protected fun addEnv(env: Map) { + if (this::constVar.isInitialized) { + var errFlag = false + env.forEach { (key, _) -> + if (key in constVar) { + LoggerService.addErrorLine("Variable $key is read-only and cannot be modified.") + errFlag = true + } + } + if (errFlag) { + throw TaskExecuteException( + errorMsg = "[Finish task] status: false, errorType: ${ErrorType.USER.num}, " + + "errorCode: ${ErrorCode.USER_INPUT_INVAILD}, message: read-only cannot be modified.", + errorType = ErrorType.USER, + errorCode = ErrorCode.USER_INPUT_INVAILD + ) + } + } environment.putAll(env) } diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/market/MarketAtomTask.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/market/MarketAtomTask.kt index 2c7b26bce50..7b86668ae9b 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/market/MarketAtomTask.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/market/MarketAtomTask.kt @@ -57,6 +57,7 @@ import com.tencent.devops.common.pipeline.EnvReplacementParser import com.tencent.devops.common.pipeline.container.VMBuildContainer import com.tencent.devops.common.pipeline.enums.BuildStatus import com.tencent.devops.common.service.utils.CommonUtils +import com.tencent.devops.common.webhook.pojo.code.BK_CI_RUN import com.tencent.devops.process.pojo.BuildTask import com.tencent.devops.process.pojo.BuildTemplateAcrossInfo import com.tencent.devops.process.pojo.BuildVariables @@ -68,6 +69,7 @@ import com.tencent.devops.process.utils.PIPELINE_ATOM_VERSION import com.tencent.devops.process.utils.PIPELINE_START_USER_ID import com.tencent.devops.process.utils.PIPELINE_STEP_ID import com.tencent.devops.process.utils.PIPELINE_TASK_NAME +import com.tencent.devops.process.utils.PipelineVarUtil import com.tencent.devops.store.pojo.atom.AtomEnv import com.tencent.devops.store.pojo.atom.enums.AtomStatusEnum import com.tencent.devops.store.pojo.common.ATOM_POST_ENTRY_PARAM @@ -82,6 +84,7 @@ import com.tencent.devops.worker.common.PIPELINE_SCRIPT_ATOM_CODE import com.tencent.devops.worker.common.WORKSPACE_CONTEXT import com.tencent.devops.worker.common.WORKSPACE_ENV import com.tencent.devops.worker.common.api.ApiFactory +import com.tencent.devops.worker.common.api.archive.ArchiveSDKApi import com.tencent.devops.worker.common.api.archive.ArtifactoryBuildResourceApi import com.tencent.devops.worker.common.api.atom.AtomArchiveSDKApi import com.tencent.devops.worker.common.api.atom.StoreSdkApi @@ -96,7 +99,6 @@ import com.tencent.devops.worker.common.exception.TaskExecuteExceptionDecorator import com.tencent.devops.worker.common.expression.SpecialFunctions import com.tencent.devops.worker.common.logger.LoggerService import com.tencent.devops.worker.common.service.CIKeywordsService -import com.tencent.devops.worker.common.service.RepoServiceFactory import com.tencent.devops.worker.common.task.ITask import com.tencent.devops.worker.common.task.TaskFactory import com.tencent.devops.worker.common.utils.ArchiveUtils @@ -122,6 +124,8 @@ open class MarketAtomTask : ITask() { private val storeApi = ApiFactory.create(StoreSdkApi::class) + private val archiveApi = ApiFactory.create(ArchiveSDKApi::class) + private val outputFile = "output.json" private val inputFile = "input.json" @@ -239,7 +243,12 @@ open class MarketAtomTask : ITask() { buildTask.stepId?.let { variables = variables.plus(PIPELINE_STEP_ID to it) } - val inputVariables = variables.plus(inputParams).toMutableMap() + val inputVariables = if (asCodeEnabled) { + // 如果开启PAC,插件入参增加旧变量，防止开启PAC后,插件获取参数失败 + PipelineVarUtil.mixOldVarAndNewVar(variables.toMutableMap()) + } else { + variables + }.plus(inputParams).toMutableMap() val atomSensitiveConfWriteSwitch = System.getProperty("BK_CI_ATOM_PRIVATE_CONFIG_WRITE_SWITCH")?.toBoolean() if (atomSensitiveConfWriteSwitch != false) { // 开关关闭则不再写入插件私有配置到input.json中 @@ -508,6 +517,7 @@ open class MarketAtomTask : ITask() { } } catch (e: Throwable) { logger.error("plugin input illegal! ", e) + LoggerService.addErrorLine("plugin input illegal! ${e.message}") throw TaskExecuteException( errorMsg = "plugin input illegal", errorType = ErrorType.SYSTEM, @@ -597,7 +607,8 @@ open class MarketAtomTask : ITask() { vmSeqId = buildTask.vmSeqId, gateway = AgentEnv.getGateway(), fileGateway = getFileGateway(buildVariables.containerType), - taskId = buildTask.taskId ?: "" + taskId = buildTask.taskId ?: "", + executeCount = buildTask.executeCount ?: 1 ) } BuildType.WORKER -> { @@ -610,7 +621,8 @@ open class MarketAtomTask : ITask() { vmSeqId = buildTask.vmSeqId, gateway = AgentEnv.getGateway(), fileGateway = getFileGateway(buildVariables.containerType), - taskId = buildTask.taskId ?: "" + taskId = buildTask.taskId ?: "", + executeCount = buildTask.executeCount ?: 1 ) } } @@ -672,14 +684,14 @@ open class MarketAtomTask : ITask() { val buildId: String, val vmSeqId: String, val fileGateway: String, - val taskId: String + val taskId: String, + val executeCount: Int ) private fun writeInputFile( workspace: File, inputVariables: Map ) { -// logger.info("runtimeVariables is:$runtimeVariables") // 有敏感信息 val inputFileFile = File(workspace, inputFile) inputFileFile.writeText(JsonUtil.toJson(inputVariables)) } @@ -794,7 +806,11 @@ open class MarketAtomTask : ITask() { val contextKey = "jobs.${buildVariables.jobId}.steps.${buildTask.stepId}.outputs.$key" env[contextKey] = value // 原变量名输出只在未开启 pipeline as code 的逻辑中保留 - if (buildVariables.pipelineAsCodeSettings?.enable == true) env.remove(key) + if ( + // TODO 暂时只对stream进行拦截原key + buildVariables.variables[BK_CI_RUN] == "true" && + buildVariables.pipelineAsCodeSettings?.enable == true + ) env.remove(key) } TaskUtil.removeTaskId() @@ -882,7 +898,7 @@ open class MarketAtomTask : ITask() { var oneArtifact = "" val artifactoryType = (output[ARTIFACTORY_TYPE] as? String) ?: ArtifactoryType.PIPELINE.name val customFlag = artifactoryType == ArtifactoryType.CUSTOM_DIR.name - val token = RepoServiceFactory.getInstance().getRepoToken( + val token = archiveApi.getRepoToken( userId = buildVariables.variables[PIPELINE_START_USER_ID] ?: "", projectId = buildVariables.projectId, repoName = if (customFlag) "custom" else "pipeline", diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/script/ICommand.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/script/ICommand.kt index e9ae1c86c23..23f7f94753a 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/script/ICommand.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/script/ICommand.kt @@ -30,6 +30,7 @@ package com.tencent.devops.worker.common.task.script import com.tencent.devops.common.api.util.KeyReplacement import com.tencent.devops.common.api.util.ReplacementUtils import com.tencent.devops.common.pipeline.EnvReplacementParser +import com.tencent.devops.process.utils.PipelineVarUtil import com.tencent.devops.store.pojo.app.BuildEnv import com.tencent.devops.worker.common.CI_TOKEN_CONTEXT import com.tencent.devops.worker.common.JOB_OS_CONTEXT @@ -79,7 +80,9 @@ interface ICommand { CI_TOKEN_CONTEXT to (variables[CI_TOKEN_CONTEXT] ?: ""), JOB_OS_CONTEXT to AgentEnv.getOS().name ) - ) + ).toMutableMap() + // 增加上下文的替换 + PipelineVarUtil.fillContextVarMap(contextMap) return if (asCodeEnabled == true) { EnvReplacementParser.parse( value = command, diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/script/ScriptTask.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/script/ScriptTask.kt index 94658c1d05a..e24f5fe3c51 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/script/ScriptTask.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/task/script/ScriptTask.kt @@ -40,12 +40,12 @@ import com.tencent.devops.process.pojo.BuildVariables import com.tencent.devops.process.utils.PIPELINE_START_USER_ID import com.tencent.devops.store.pojo.app.BuildEnv import com.tencent.devops.worker.common.api.ApiFactory +import com.tencent.devops.worker.common.api.archive.ArchiveSDKApi import com.tencent.devops.worker.common.api.quality.QualityGatewaySDKApi import com.tencent.devops.worker.common.constants.WorkerMessageCode.BK_NO_FILES_TO_ARCHIVE import com.tencent.devops.worker.common.constants.WorkerMessageCode.SCRIPT_EXECUTION_FAIL import com.tencent.devops.worker.common.env.AgentEnv import com.tencent.devops.worker.common.logger.LoggerService -import com.tencent.devops.worker.common.service.RepoServiceFactory import com.tencent.devops.worker.common.task.ITask import com.tencent.devops.worker.common.task.script.bat.WindowsScriptTask import com.tencent.devops.worker.common.utils.ArchiveUtils @@ -62,6 +62,7 @@ import java.net.URLDecoder open class ScriptTask : ITask() { private val gatewayResourceApi = ApiFactory.create(QualityGatewaySDKApi::class) + private val archiveApi = ApiFactory.create(ArchiveSDKApi::class) override fun execute(buildTask: BuildTask, buildVariables: BuildVariables, workspace: File) { val taskParams = buildTask.params ?: mapOf() @@ -130,7 +131,7 @@ open class ScriptTask : ITask() { arrayOf(archiveFileIfExecFail) ) ) - val token = RepoServiceFactory.getInstance().getRepoToken( + val token = archiveApi.getRepoToken( userId = buildVariables.variables[PIPELINE_START_USER_ID] ?: "", projectId = buildVariables.projectId, repoName = "pipeline", diff --git a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/utils/CommandLineUtils.kt b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/utils/CommandLineUtils.kt index edfbf4422cf..60a7aee3290 100644 --- a/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/utils/CommandLineUtils.kt +++ b/src/backend/ci/core/worker/worker-common/src/main/kotlin/com/tencent/devops/worker/common/utils/CommandLineUtils.kt @@ -172,22 +172,24 @@ object CommandLineUtils { return result.toString() } - private fun reportProgressRate( + fun reportProgressRate( taskId: String?, tmpLine: String - ) { - val pattern = Pattern.compile("::set-progress-rate\\s*(.*)") - val matcher = pattern.matcher(tmpLine) + ): Double? { + val pattern = Pattern.compile("^[\"]?::set-progress-rate\\s*(.*)$") + val matcher = pattern.matcher(tmpLine.trim()) if (matcher.find()) { - val progressRate = matcher.group(1) - if (taskId != null) { + val progressRate = matcher.group(1).removeSuffix("\"").toDoubleOrNull() + if (taskId != null && progressRate != null) { Heartbeat.recordTaskProgressRate( taskId = taskId, - progressRate = progressRate.toDouble() + progressRate = progressRate ) } logger.info("report progress rate:$tmpLine|$taskId|$progressRate") + return progressRate } + return null } private fun appendResultToFile( @@ -210,46 +212,48 @@ object CommandLineUtils { appendOutputToFile(tmpLine, workspace, resultLogFile, jobId, stepId) } - private fun appendVariableToFile( + fun appendVariableToFile( tmpLine: String, workspace: File?, resultLogFile: String - ) { + ): String? { val pattenVar = "[\"]?::set-variable\\sname=.*" val prefixVar = "::set-variable name=" if (Pattern.matches(pattenVar, tmpLine)) { val value = tmpLine.removeSurrounding("\"").removePrefix(prefixVar) val keyValue = value.split("::") if (keyValue.size >= 2) { - File(workspace, resultLogFile).appendText( - "variables.${keyValue[0]}=${value.removePrefix("${keyValue[0]}::")}\n" - ) + val res = "variables.${keyValue[0]}=${value.removePrefix("${keyValue[0]}::")}\n" + File(workspace, resultLogFile).appendText(res) + return res } } + return null } - private fun appendRemarkToFile( + fun appendRemarkToFile( tmpLine: String, workspace: File?, resultLogFile: String - ) { + ): String? { val pattenVar = "[\"]?::set-remark\\s.*" val prefixVar = "::set-remark " if (Pattern.matches(pattenVar, tmpLine)) { val value = tmpLine.removeSurrounding("\"").removePrefix(prefixVar) - File(workspace, resultLogFile).appendText( - "BK_CI_BUILD_REMARK=$value\n" - ) + val res = "BK_CI_BUILD_REMARK=$value\n" + File(workspace, resultLogFile).appendText(res) + return res } + return null } - private fun appendOutputToFile( + fun appendOutputToFile( tmpLine: String, workspace: File?, resultLogFile: String, jobId: String, stepId: String - ) { + ): String? { val pattenOutput = "[\"]?::set-output\\sname=.*" val prefixOutput = "::set-output name=" if (Pattern.matches(pattenOutput, tmpLine)) { @@ -257,29 +261,31 @@ object CommandLineUtils { val keyValue = value.split("::") val keyPrefix = "jobs.$jobId.steps.$stepId.outputs." if (keyValue.size >= 2) { - File(workspace, resultLogFile).appendText( - "$keyPrefix${keyValue[0]}=${value.removePrefix("${keyValue[0]}::")}\n" - ) + val res = "$keyPrefix${keyValue[0]}=${value.removePrefix("${keyValue[0]}::")}\n" + File(workspace, resultLogFile).appendText(res) + return res } } + return null } - private fun appendGateToFile( + fun appendGateToFile( tmpLine: String, workspace: File?, resultLogFile: String - ) { + ): String? { val pattenOutput = "[\"]?::set-gate-value\\sname=.*" val prefixOutput = "::set-gate-value name=" if (Pattern.matches(pattenOutput, tmpLine)) { val value = tmpLine.removeSurrounding("\"").removePrefix(prefixOutput) val keyValue = value.split("::") if (keyValue.size >= 2) { - File(workspace, resultLogFile).appendText( - "${keyValue[0]}=${value.removePrefix("${keyValue[0]}::")}\n" - ) + val res = "${keyValue[0]}=${value.removePrefix("${keyValue[0]}::")}\n" + File(workspace, resultLogFile).appendText(res) + return res } } + return null } fun execute(file: File, workspace: File?, print2Logger: Boolean, prefix: String = ""): String { diff --git a/src/backend/dispatch-k8s-manager/Makefile b/src/backend/dispatch-k8s-manager/Makefile index 8c4d320af44..098ce1af5d3 100644 --- a/src/backend/dispatch-k8s-manager/Makefile +++ b/src/backend/dispatch-k8s-manager/Makefile @@ -13,6 +13,9 @@ GOFLAGS := # gin export GIN_MODE=release +format: + find ./ -name "*.go" | xargs gofmt -w + test: test-unit .PHONY: test-unit diff --git a/src/backend/dispatch-k8s-manager/cmd/apiserver/apiserver.go b/src/backend/dispatch-k8s-manager/cmd/apiserver/apiserver.go index 72cc958334e..e543b6f0615 100644 --- a/src/backend/dispatch-k8s-manager/cmd/apiserver/apiserver.go +++ b/src/backend/dispatch-k8s-manager/cmd/apiserver/apiserver.go @@ -4,9 +4,11 @@ import ( "disaptch-k8s-manager/pkg/apiserver" "disaptch-k8s-manager/pkg/buildless" "disaptch-k8s-manager/pkg/config" + "disaptch-k8s-manager/pkg/constant" "disaptch-k8s-manager/pkg/cron" "disaptch-k8s-manager/pkg/db/mysql" "disaptch-k8s-manager/pkg/db/redis" + "disaptch-k8s-manager/pkg/docker" "disaptch-k8s-manager/pkg/kubeclient" "disaptch-k8s-manager/pkg/logs" "disaptch-k8s-manager/pkg/task" @@ -63,6 +65,10 @@ func main() { os.Exit(1) } + if config.Config.Docker.Enable { + docker.InitDockerCli() + } + if err := apiserver.InitApiServer(filepath.Join(outDir, "logs", config.AccessLog)); err != nil { fmt.Printf("init api server error %v\n", err) os.Exit(1) @@ -72,7 +78,7 @@ func main() { } func initConfig(configDir string) { - if debug == "true" { + if debug == "true" || os.Getenv(constant.KubernetesManagerDebugEnable) == "true" { config.Envs.IsDebug = true } else { config.Envs.IsDebug = false diff --git a/src/backend/dispatch-k8s-manager/go.mod b/src/backend/dispatch-k8s-manager/go.mod index b9803b9a281..92a0debe35f 100644 --- a/src/backend/dispatch-k8s-manager/go.mod +++ b/src/backend/dispatch-k8s-manager/go.mod @@ -1,23 +1,25 @@ module disaptch-k8s-manager -go 1.18 +go 1.19 require ( - github.com/gin-gonic/gin v1.8.1 - github.com/go-playground/locales v0.14.0 - github.com/go-playground/universal-translator v0.18.0 - github.com/go-playground/validator/v10 v10.11.0 + github.com/docker/docker v24.0.1+incompatible + github.com/gin-gonic/gin v1.9.1 + github.com/go-playground/locales v0.14.1 + github.com/go-playground/universal-translator v0.18.1 + github.com/go-playground/validator/v10 v10.14.0 github.com/go-redis/redis v6.15.9+incompatible github.com/go-sql-driver/mysql v1.6.0 github.com/gorilla/websocket v1.4.2 github.com/pkg/errors v0.9.1 github.com/robfig/cron/v3 v3.0.0 - github.com/sirupsen/logrus v1.8.1 + github.com/sirupsen/logrus v1.9.0 github.com/spf13/viper v1.11.0 - github.com/stretchr/testify v1.8.0 + github.com/stretchr/testify v1.8.3 github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe github.com/swaggo/gin-swagger v1.5.1 - github.com/swaggo/swag v1.8.4 + github.com/swaggo/swag v1.16.1 + golang.org/x/net v0.23.0 gopkg.in/natefinch/lumberjack.v2 v2.0.0 k8s.io/api v0.24.0 k8s.io/apimachinery v0.24.0 @@ -25,19 +27,25 @@ require ( ) require ( + github.com/BurntSushi/toml v1.2.1 // indirect github.com/KyleBanks/depth v1.2.1 // indirect - github.com/PuerkitoBio/purell v1.1.1 // indirect - github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/bytedance/sonic v1.9.1 // indirect + github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/docker/distribution v2.8.2+incompatible // indirect + github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-units v0.5.0 // indirect github.com/emicklei/go-restful v2.16.0+incompatible // indirect github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/gabriel-vasile/mimetype v1.4.2 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-logr/logr v1.2.0 // indirect - github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.19.6 // indirect - github.com/go-openapi/spec v0.20.4 // indirect - github.com/go-openapi/swag v0.19.15 // indirect - github.com/goccy/go-json v0.9.7 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/spec v0.20.9 // indirect + github.com/go-openapi/swag v0.22.3 // indirect + github.com/goccy/go-json v0.10.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect @@ -47,41 +55,49 @@ require ( github.com/imdario/mergo v0.3.5 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/leodido/go-urn v1.2.1 // indirect + github.com/klauspost/cpuid/v2 v2.2.4 // indirect + github.com/leodido/go-urn v1.2.4 // indirect github.com/magiconair/properties v1.8.6 // indirect - github.com/mailru/easyjson v0.7.6 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/mitchellh/mapstructure v1.4.3 // indirect + github.com/moby/term v0.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/morikuni/aec v1.0.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.0.2 // indirect github.com/pelletier/go-toml v1.9.4 // indirect - github.com/pelletier/go-toml/v2 v2.0.1 // indirect + github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/spf13/afero v1.8.2 // indirect github.com/spf13/cast v1.4.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.2.0 // indirect - github.com/ugorji/go/codec v1.2.7 // indirect - golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect - golang.org/x/net v0.7.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.11 // indirect + golang.org/x/arch v0.3.0 // indirect + golang.org/x/crypto v0.21.0 // indirect + golang.org/x/mod v0.10.0 // indirect golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/term v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect - golang.org/x/tools v0.1.12 // indirect + golang.org/x/tools v0.9.1 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.28.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.66.4 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + gotest.tools/v3 v3.4.0 // indirect k8s.io/klog/v2 v2.60.1 // indirect k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42 // indirect k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect - sigs.k8s.io/yaml v1.2.0 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/src/backend/dispatch-k8s-manager/go.sum b/src/backend/dispatch-k8s-manager/go.sum index 91cd0f1b950..fe6d479eb48 100644 --- a/src/backend/dispatch-k8s-manager/go.sum +++ b/src/backend/dispatch-k8s-manager/go.sum @@ -39,6 +39,7 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= @@ -46,20 +47,27 @@ github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSY github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/agiledragon/gomonkey/v2 v2.3.1/go.mod h1:ap1AmDzcVOAz1YpeJ3TCzIgstoaWLA6jbbgxfB4w2iY= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= +github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s= +github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= +github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 h1:qSGYFH7+jGhDF8vLC+iwCD4WpbV1EBDSzWkJODFLams= +github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311/go.mod h1:b583jCggY9gE99b6G5LEC39OIiVsWj+R97kbl5odCEk= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -72,6 +80,14 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= +github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v24.0.1+incompatible h1:NxN81beIxDlUaVt46iUQrYHD9/W3u9EGl52r86O/IGw= +github.com/docker/docker v24.0.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= @@ -91,14 +107,17 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= +github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/gzip v0.0.6 h1:NjcunTcGAj5CO1gn4N8jHOSIeRFHIbn51z6K+xaN4d4= github.com/gin-contrib/gzip v0.0.6/go.mod h1:QOJlmV2xmayAjkNS2Y8NQsMneuRShOU/kjovCXNuzzk= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.8.1 h1:4+fr/el88TOO3ewCmQr8cx/CtZ/umlIRIs5M4NTNjf8= github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= +github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= +github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -107,33 +126,41 @@ github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg github.com/go-logr/logr v1.2.0 h1:QK40JKJyMdUDz+h+xvCsru/bJhvG0UxvePV0ufL/AcE= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= -github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs= github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= -github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M= +github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= +github.com/go-openapi/spec v0.20.9 h1:xnlYNQAwKd2VQRRfwTEI0DcK+2cbuvI/0c7jx3gA8/8= +github.com/go-openapi/spec v0.20.9/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= +github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= -github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= -github.com/go-playground/validator/v10 v10.11.0 h1:0W+xRM511GY47Yy3bZUbJVitCNg2BOGlCyvTqsp/xIw= -github.com/go-playground/validator/v10 v10.11.0/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= +github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= +github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/goccy/go-json v0.9.7 h1:IcB+Aqpx/iMHu5Yooh7jEzJk1JZ7Pjtmys2ukPr7EeM= github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -233,6 +260,9 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= +github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -243,26 +273,33 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= +github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= +github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= @@ -279,6 +316,10 @@ github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGV github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= +github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/otiai10/copy v1.7.0/go.mod h1:rmRl6QPdJj6EiUqXQ/4Nn2lLXoNQjFCQbbNrxgc/t3U= github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= @@ -286,8 +327,9 @@ github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT9 github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.0.1 h1:8e3L2cCQzLFi2CR4g7vGFuFxX7Jl1kKX8gW+iV0GUKU= github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= +github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= +github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -304,8 +346,8 @@ github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUA github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= @@ -322,6 +364,7 @@ github.com/spf13/viper v1.11.0/go.mod h1:djo0X/bA5+tYVoCn+C7cAYJGcVn/qYLFTG8gdUs github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -329,8 +372,11 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe h1:K8pHPVoTgxFJt1lXuIzzOX7zZhZFldJQK/CgKx9BFIc= @@ -338,11 +384,14 @@ github.com/swaggo/files v0.0.0-20220610200504-28940afbdbfe/go.mod h1:lKJPbtWzJ9J github.com/swaggo/gin-swagger v1.5.1 h1:PFmlJU1LPn8DjrR0meVLX5gyFdgcPOkLcoFRRFx7WcY= github.com/swaggo/gin-swagger v1.5.1/go.mod h1:Cbj/MlHApPOjZdf4joWFXLLgmZVPyh54GPvPPyVjVZM= github.com/swaggo/swag v1.8.1/go.mod h1:ugemnJsPZm/kRwFUnzBlbHRd0JY9zE1M4F+uy2pAaPQ= -github.com/swaggo/swag v1.8.4 h1:oGB351qH1JqUqK1tsMYEE5qTBbPk394BhsZxmUfebcI= -github.com/swaggo/swag v1.8.4/go.mod h1:jMLeXOOmYyjk8PvHTsXBdrubsNd9gUJTTCzL5iBnseg= +github.com/swaggo/swag v1.16.1 h1:fTNRhKstPKxcnoKsytm4sahr8FaYzUcT7i1/3nd/fBg= +github.com/swaggo/swag v1.16.1/go.mod h1:9/LMvHycG3NFHfR6LwvikHv5iFvmPADQ359cKikGxto= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= -github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= +github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -357,6 +406,9 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= +golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= +golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -366,10 +418,9 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 h1:kUhD7nTDoI3fVd9G4ORWrbV5NY0liEs/Jg2pv5f+bBA= -golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -404,7 +455,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= +golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -448,8 +500,8 @@ golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -475,6 +527,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -488,7 +541,6 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -528,12 +580,15 @@ golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -543,8 +598,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -603,8 +658,8 @@ golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo= -golang.org/x/tools v0.1.12 h1:VveCTK38A2rkS8ZqFY25HIDFscX5X9OoEhJd3quQmXU= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= +golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -712,8 +767,9 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -743,6 +799,8 @@ gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.4.0 h1:ZazjZUfuVeZGLAmlKKuyv3IKP5orXcwtOwDQH6YVr6o= +gotest.tools/v3 v3.4.0/go.mod h1:CtbdzLSsqVhDgMtKsx03ird5YTGB3ar27v0u/yKBW5g= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -767,6 +825,7 @@ k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 h1:HNSDgDCrr/6Ly3WEGKZftiE7IY19Vz2GdbOCyI4qqhc= k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 h1:kDi4JBNAsJWfz1aEXhO8Jg87JJaPNLh5tIzYHgStQ9Y= @@ -774,5 +833,6 @@ sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/src/backend/dispatch-k8s-manager/pkg/apiserver/apis/apis.go b/src/backend/dispatch-k8s-manager/pkg/apiserver/apis/apis.go index 90dea83bb4d..9b645adf0d4 100644 --- a/src/backend/dispatch-k8s-manager/pkg/apiserver/apis/apis.go +++ b/src/backend/dispatch-k8s-manager/pkg/apiserver/apis/apis.go @@ -37,6 +37,11 @@ func InitApis(r *gin.Engine, handlers ...gin.HandlerFunc) { initBuilderApis(apis) initTasksApis(apis) initBuildLessApis(apis) + initDeploymentApis(apis) + initServiceApis(apis) + initIngressApis(apis) + initSecretApis(apis) + initDockerApis(apis) } func ok(c *gin.Context, data interface{}) { diff --git a/src/backend/dispatch-k8s-manager/pkg/apiserver/apis/docker.go b/src/backend/dispatch-k8s-manager/pkg/apiserver/apis/docker.go new file mode 100644 index 00000000000..c065d92cd2c --- /dev/null +++ b/src/backend/dispatch-k8s-manager/pkg/apiserver/apis/docker.go @@ -0,0 +1,44 @@ +package apis + +import ( + "disaptch-k8s-manager/pkg/apiserver/service" + "net/http" + + "github.com/gin-gonic/gin" +) + +const ( + dockerPrefix = "/docker" +) + +func initDockerApis(r *gin.RouterGroup) { + docker := r.Group(dockerPrefix) + { + docker.POST("/inspect", dockerInspect) + } +} + +// @Tags docker +// @Summary docker inspect命令(同时会pull) +// @Accept json +// @Product json +// @Param Devops-Token header string true "凭证信息" +// @Param info body service.DockerInspectInfo true "构建机信息" +// @Success 200 {object} types.Result{data=service.TaskId} "任务ID" +// @Router /docker/inspect [post] +func dockerInspect(c *gin.Context) { + info := &service.DockerInspectInfo{} + + if err := c.BindJSON(info); err != nil { + fail(c, http.StatusBadRequest, err) + return + } + + taskId, err := service.DockerInspect(info) + if err != nil { + fail(c, http.StatusInternalServerError, err) + return + } + + ok(c, service.TaskId{TaskId: taskId}) +} diff --git a/src/backend/dispatch-k8s-manager/pkg/apiserver/apis/open_deployment.go b/src/backend/dispatch-k8s-manager/pkg/apiserver/apis/open_deployment.go new file mode 100644 index 00000000000..f52b38fa388 --- /dev/null +++ b/src/backend/dispatch-k8s-manager/pkg/apiserver/apis/open_deployment.go @@ -0,0 +1,121 @@ +package apis + +import ( + "disaptch-k8s-manager/pkg/kubeclient" + "disaptch-k8s-manager/pkg/logs" + "fmt" + "github.com/gin-gonic/gin" + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + "net/http" +) + +const ( + deploymentPrefix = "/namespace/:namespace/deployments" +) + +func initDeploymentApis(r *gin.RouterGroup) { + deployment := r.Group(deploymentPrefix) + { + deployment.GET("/:deploymentName", getDeployment) + deployment.POST("", createDeployment) + deployment.DELETE("/:deploymentName", deleteDeployment) + } +} + +// @Tags deployment +// @Summary 获取deployment状态 +// @Accept json +// @Product json +// @Param Devops-Token header string true "凭证信息" +// @Param deploymentName path string true "deployment名称" +// @Success 200 {object} types.Result{data=appsv1.Deployment} "deployment详情" +// @Router /deployment/{deploymentName} [get] +func getDeployment(c *gin.Context) { + namespace := c.Param("namespace") + deploymentName := c.Param("deploymentName") + + if !checkDeploymentParams(c, deploymentName) { + return + } + + deployment, err := kubeclient.GetNativeDeployment(namespace, deploymentName) + if err != nil { + okFail(c, http.StatusInternalServerError, err) + return + } + + ok(c, deployment) +} + +// @Tags deployment +// @Summary 创建deployment负载资源 +// @Accept json +// @Product json +// @Param Devops-Token header string true "凭证信息" +// @Param deployment body appsv1.Deployment true "deployment负载信息" +// @Success 200 {object} "" +// @Router /deployment [post] +func createDeployment(c *gin.Context) { + namespace := c.Param("namespace") + + deployment := &appsv1.Deployment{} + + if err := c.BindJSON(deployment); err != nil { + fail(c, http.StatusBadRequest, err) + return + } + + deploymentInfo, _ := kubeclient.GetNativeDeployment(namespace, deployment.Name) + if deploymentInfo != nil { + logs.Info(fmt.Sprintf("Deployment: %s exist, update.", deployment.Name)) + updateErr := kubeclient.UpdateNativeDeployment(namespace, deployment) + if updateErr != nil { + fail(c, http.StatusInternalServerError, updateErr) + return + } + } else { + logs.Info(fmt.Sprintf("Deployment: %s not exist, create.", deployment.Name)) + createErr := kubeclient.CreateNativeDeployment(namespace, deployment) + if createErr != nil { + fail(c, http.StatusInternalServerError, createErr) + return + } + } + + ok(c, "") +} + +// @Tags deployment +// @Summary 删除deployment +// @Accept json +// @Product json +// @Param Devops-Token header string true "凭证信息" +// @Param deploymentName path string true "deployment名称" +// @Success 200 {object} types.Result{data=""} "" +// @Router /deployment/{deploymentName} [delete] +func deleteDeployment(c *gin.Context) { + namespace := c.Param("namespace") + deploymentName := c.Param("deploymentName") + + if !checkDeploymentParams(c, deploymentName) { + return + } + + err := kubeclient.DeleteNativeDeployment(namespace, deploymentName) + if err != nil { + fail(c, http.StatusInternalServerError, err) + return + } + + ok(c, "") +} + +func checkDeploymentParams(c *gin.Context, deploymentName string) bool { + if deploymentName == "" { + fail(c, http.StatusBadRequest, errors.New("deployment名称不能为空")) + return false + } + + return true +} diff --git a/src/backend/dispatch-k8s-manager/pkg/apiserver/apis/open_ingress.go b/src/backend/dispatch-k8s-manager/pkg/apiserver/apis/open_ingress.go new file mode 100644 index 00000000000..055e5bf9343 --- /dev/null +++ b/src/backend/dispatch-k8s-manager/pkg/apiserver/apis/open_ingress.go @@ -0,0 +1,116 @@ +package apis + +import ( + "disaptch-k8s-manager/pkg/kubeclient" + "github.com/gin-gonic/gin" + "github.com/pkg/errors" + networkv1 "k8s.io/api/networking/v1" + "net/http" +) + +const ( + ingressPrefix = "/namespace/:namespace/ingress" +) + +func initIngressApis(r *gin.RouterGroup) { + ingress := r.Group(ingressPrefix) + { + ingress.GET("/:ingressName", getIngress) + ingress.POST("", createIngress) + ingress.DELETE("/:ingressName", deleteIngress) + } +} + +// @Tags ingress +// @Summary 获取ingress详情 +// @Accept json +// @Product json +// @Param Devops-Token header string true "凭证信息" +// @Param ingressName path string true "ingress名称" +// @Success 200 {object} types.Result{data=networkv1.ingress} "ingress详情" +// @Router /ingress/{ingressName} [get] +func getIngress(c *gin.Context) { + namespace := c.Param("namespace") + ingressName := c.Param("ingressName") + + if !checkIngressName(c, ingressName) { + return + } + + ingress, err := kubeclient.GetIngress(namespace, ingressName) + if err != nil { + okFail(c, http.StatusInternalServerError, err) + return + } + + ok(c, ingress) +} + +// @Tags ingress +// @Summary 创建ingress负载资源 +// @Accept json +// @Product json +// @Param Devops-Token header string true "凭证信息" +// @Param ingress body networkv1.ingress true "ingress负载信息" +// @Success 200 {object} "" +// @Router /ingress [post] +func createIngress(c *gin.Context) { + namespace := c.Param("namespace") + ingress := &networkv1.Ingress{} + + if err := c.BindJSON(ingress); err != nil { + fail(c, http.StatusBadRequest, err) + return + } + + ingressInfo, _ := kubeclient.GetIngress(namespace, ingress.Name) + if ingressInfo != nil { + err := kubeclient.UpdateIngress(namespace, ingress) + if err != nil { + fail(c, http.StatusInternalServerError, err) + return + } + } else { + err := kubeclient.CreateIngress(namespace, ingress) + if err != nil { + fail(c, http.StatusInternalServerError, err) + return + } + } + + ok(c, "") +} + +// @Tags ingress +// @Summary 删除ingress +// @Accept json +// @Product json +// @Param Devops-Token header string true "凭证信息" +// @Param ingressName path string true "ingress名称" +// @Success 200 {object} types.Result{data=""} "" +// @Router /ingress/{ingressName} [delete] +func deleteIngress(c *gin.Context) { + namespace := c.Param("namespace") + ingressName := c.Param("ingressName") + + if !checkIngressName(c, ingressName) { + return + } + + err := kubeclient.DeleteIngress(namespace, ingressName) + if err != nil { + fail(c, http.StatusInternalServerError, err) + return + } + + ok(c, "") +} + +func checkIngressName(c *gin.Context, deploymentName string) bool { + if deploymentName == "" { + fail(c, http.StatusBadRequest, errors.New("ingress名称不能为空")) + return false + } + + return true +} diff --git a/src/backend/dispatch-k8s-manager/pkg/apiserver/apis/open_secret.go b/src/backend/dispatch-k8s-manager/pkg/apiserver/apis/open_secret.go new file mode 100644 index 00000000000..9a3369d1106 --- /dev/null +++ b/src/backend/dispatch-k8s-manager/pkg/apiserver/apis/open_secret.go @@ -0,0 +1,107 @@ +package apis + +import ( + "disaptch-k8s-manager/pkg/kubeclient" + "github.com/gin-gonic/gin" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + "net/http" +) + +const ( + secretPrefix = "/namespace/:namespace/secrets" +) + +func initSecretApis(r *gin.RouterGroup) { + secret := r.Group(secretPrefix) + { + secret.GET("/:secretName", getSecret) + secret.POST("", createSecret) + secret.DELETE("/:secretName", deleteSecret) + } +} + +// @Tags secret +// @Summary 获取secret详情 +// @Accept json +// @Product json +// @Param Devops-Token header string true "凭证信息" +// @Param secretName path string true "secret名称" +// @Success 200 {object} types.Result{data=corev1.secret} "secret详情" +// @Router /secret/{secretName} [get] +func getSecret(c *gin.Context) { + namespace := c.Param("namespace") + secretName := c.Param("secretName") + + if !checkSecretName(c, secretName) { + return + } + + secret, err := kubeclient.GetNativeSecret(namespace, secretName) + if err != nil { + okFail(c, http.StatusInternalServerError, err) + return + } + + ok(c, secret) +} + +// @Tags secret +// @Summary 创建secret负载资源 +// @Accept json +// @Product json +// @Param Devops-Token header string true "凭证信息" +// @Param secret body corev1.secret true "secret信息" +// @Success 200 {object} "" +// @Router /secret [post] +func createSecret(c *gin.Context) { + namespace := c.Param("namespace") + secret := &corev1.Secret{} + + if err := c.BindJSON(secret); err != nil { + fail(c, http.StatusBadRequest, err) + return + } + + err := kubeclient.CreateNativeSecret(namespace, secret) + if err != nil { + fail(c, http.StatusInternalServerError, err) + return + } + + ok(c, "") +} + +// @Tags secret +// @Summary 删除secret +// @Accept json +// @Product json +// @Param Devops-Token header string true "凭证信息" +// @Param secretName path string true "secret名称" +// @Success 200 {object} types.Result{data=""} "" +// @Router /secret/{secretName} [delete] +func deleteSecret(c *gin.Context) { + namespace := c.Param("namespace") + secretName := c.Param("secretName") + + if !checkSecretName(c, secretName) { + return + } + + err := kubeclient.DeleteNativeSecret(namespace, secretName) + if err != nil { + fail(c, http.StatusInternalServerError, err) + return + } + + ok(c, "") +} + +func checkSecretName(c *gin.Context, secretName string) bool { + if secretName == "" { + fail(c, http.StatusBadRequest, errors.New("secret名称不能为空")) + return false + } + + return true +} diff --git a/src/backend/dispatch-k8s-manager/pkg/apiserver/apis/open_service.go b/src/backend/dispatch-k8s-manager/pkg/apiserver/apis/open_service.go new file mode 100644 index 00000000000..289df57b985 --- /dev/null +++ b/src/backend/dispatch-k8s-manager/pkg/apiserver/apis/open_service.go @@ -0,0 +1,116 @@ +package apis + +import ( + "disaptch-k8s-manager/pkg/kubeclient" + "github.com/gin-gonic/gin" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + "net/http" +) + +const ( + servicePrefix = "/namespace/:namespace/services" +) + +func initServiceApis(r *gin.RouterGroup) { + service := r.Group(servicePrefix) + { + service.GET("/:serviceName", getService) + service.POST("", createService) + service.DELETE("/:serviceName", deleteService) + } +} + +// @Tags service +// @Summary 获取service详情 +// @Accept json +// @Product json +// @Param Devops-Token header string true "凭证信息" +// @Param serviceName path string true "deployment名称" +// @Success 200 {object} types.Result{data=appsv1.service} "service详情" +// @Router /service/{serviceName} [get] +func getService(c *gin.Context) { + namespace := c.Param("namespace") + serviceName := c.Param("serviceName") + + if !checkServiceName(c, serviceName) { + return + } + + service, err := kubeclient.GetService(namespace, serviceName) + if err != nil { + okFail(c, http.StatusInternalServerError, err) + return + } + + ok(c, service) +} + +// @Tags service +// @Summary 创建service负载资源 +// @Accept json +// @Product json +// @Param Devops-Token header string true "凭证信息" +// @Param service body corev1.Service true "service负载信息" +// @Success 200 {object} "" +// @Router /service [post] +func createService(c *gin.Context) { + namespace := c.Param("namespace") + service := &corev1.Service{} + + if err := c.BindJSON(service); err != nil { + fail(c, http.StatusBadRequest, err) + return + } + + serviceInfo, _ := kubeclient.GetService(namespace, service.Name) + if serviceInfo != nil { + err := kubeclient.UpdateService(namespace, service) + if err != nil { + fail(c, http.StatusInternalServerError, err) + return + } + } else { + err := kubeclient.CreateService(namespace, service) + if err != nil { + fail(c, http.StatusInternalServerError, err) + return + } + } + + ok(c, "") +} + +// @Tags service +// @Summary 删除service +// @Accept json +// @Product json +// @Param Devops-Token header string true "凭证信息" +// @Param serviceName path string true "service名称" +// @Success 200 {object} types.Result{data=""} "" +// @Router /service/{serviceName} [delete] +func deleteService(c *gin.Context) { + namespace := c.Param("namespace") + serviceName := c.Param("serviceName") + + if !checkServiceName(c, serviceName) { + return + } + + err := kubeclient.DeleteService(namespace, serviceName) + if err != nil { + fail(c, http.StatusInternalServerError, err) + return + } + + ok(c, "") +} + +func checkServiceName(c *gin.Context, deploymentName string) bool { + if deploymentName == "" { + fail(c, http.StatusBadRequest, errors.New("service名称不能为空")) + return false + } + + return true +} diff --git a/src/backend/dispatch-k8s-manager/pkg/apiserver/apis/task.go b/src/backend/dispatch-k8s-manager/pkg/apiserver/apis/task.go index e7bcbd6e7cd..216c0ea27ac 100644 --- a/src/backend/dispatch-k8s-manager/pkg/apiserver/apis/task.go +++ b/src/backend/dispatch-k8s-manager/pkg/apiserver/apis/task.go @@ -8,9 +8,9 @@ import ( ) func initTasksApis(r *gin.RouterGroup) { - jobs := r.Group("/tasks") + tasks := r.Group("/tasks") { - jobs.GET("/:taskId/status", getTaskStatus) + tasks .GET("/:taskId/status", getTaskStatus) } } diff --git a/src/backend/dispatch-k8s-manager/pkg/apiserver/service/docker.go b/src/backend/dispatch-k8s-manager/pkg/apiserver/service/docker.go new file mode 100644 index 00000000000..aafc2f745fa --- /dev/null +++ b/src/backend/dispatch-k8s-manager/pkg/apiserver/service/docker.go @@ -0,0 +1,111 @@ +package service + +import ( + "context" + "disaptch-k8s-manager/pkg/db/mysql" + "disaptch-k8s-manager/pkg/docker" + "disaptch-k8s-manager/pkg/logs" + "disaptch-k8s-manager/pkg/task" + "disaptch-k8s-manager/pkg/types" + "encoding/json" + "strings" + "time" + + dockerTypes "github.com/docker/docker/api/types" + "github.com/pkg/errors" +) + +func DockerInspect(info *DockerInspectInfo) (string, error) { + taskId := generateTaskId() + + if err := mysql.InsertTask(types.Task{ + TaskId: taskId, + TaskKey: info.Name, + TaskBelong: types.TaskBelongDocker, + Action: types.TaskDockerActionInspect, + Status: types.TaskWaiting, + Message: nil, + ActionTime: time.Now(), + UpdateTime: time.Now(), + }); err != nil { + return "", err + } + + go inspect(taskId, info) + + return taskId, nil +} + +func inspect(taskId string, info *DockerInspectInfo) { + task.UpdateTask(taskId, types.TaskRunning) + + ctx := context.Background() + + // 拉取镜像 + pullMsg, err := docker.ImagePull(ctx, info.Ref, info.Credential.Username, info.Credential.Password) + if err != nil { + logs.Error("inspect ImagePull error", err) + task.FailTask(taskId, err.Error()) + return + } + + // 寻找ID + imageName := strings.TrimSpace(info.Ref) + imageStr := strings.TrimPrefix(strings.TrimPrefix(imageName, "http://"), "https://") + images, err := docker.ImageList(ctx) + if err != nil { + logs.Error("get image list error", err) + task.FailTask(taskId, err.Error()) + return + } + id := "" + for _, image := range images { + for _, tagName := range image.RepoTags { + if tagName == imageStr { + id = image.ID + } + } + } + if id == "" { + err = errors.Errorf("image %s not found", imageName) + logs.Errorf("pullMsg %s error %s", pullMsg, err.Error()) + task.FailTask(taskId, err.Error()) + return + } + + defer func() { + // 完事后删除镜像 + if err = docker.ImageRemove(ctx, id, dockerTypes.ImageRemoveOptions{Force: true}); err != nil { + logs.Errorf("remove image %s id %s error %s", info.Ref, id, err.Error()) + } + }() + + // 分析镜像 + image, err := docker.ImageInspect(ctx, info.Ref) + if err != nil { + logs.Error("inspect ImageInspect error", err) + task.FailTask(taskId, err.Error()) + return + } + + msg := &DockerInspectResp{ + Architecture: image.Architecture, + Os: image.Os, + Size: image.Size, + Created: image.Created, + Id: image.ID, + Author: image.Author, + Parent: image.Parent, + OsVersion: image.OsVersion, + } + + msgStr, err := json.Marshal(msg) + if err != nil { + logs.Error("inspect jsonMarshal error", err) + task.FailTask(taskId, err.Error()) + return + } + + task.OkTaskWithMessage(taskId, string(msgStr)) + +} diff --git a/src/backend/dispatch-k8s-manager/pkg/apiserver/service/docker_type.go b/src/backend/dispatch-k8s-manager/pkg/apiserver/service/docker_type.go new file mode 100644 index 00000000000..c0e3ef7d4c0 --- /dev/null +++ b/src/backend/dispatch-k8s-manager/pkg/apiserver/service/docker_type.go @@ -0,0 +1,23 @@ +package service + +type DockerInspectInfo struct { + Name string `json:"name" binding:"required"` // 任务名称，唯一 + Ref string `json:"ref" binding:"required"` // docker镜像信息 如：docker:latest + Credential Credential `json:"cred"` // 拉取镜像凭据 +} + +type Credential struct { + Username string `json:"username"` + Password string `json:"password"` +} + +type DockerInspectResp struct { + Architecture string `json:"arch"` // 架构 + Os string `json:"os"` // 系统 + Size int64 `json:"size"` // 大小 + Created string `json:"created"` // 创建时间 + Id string `json:"id"` // id + Author string `json:"author"` // 作者 + Parent string `json:"parent"` // 父镜像信息 + OsVersion string `json:"osVersion"` // 系统版本 +} diff --git a/src/backend/dispatch-k8s-manager/pkg/config/config_type.go b/src/backend/dispatch-k8s-manager/pkg/config/config_type.go index c00c1bdd966..dd90e9d6759 100644 --- a/src/backend/dispatch-k8s-manager/pkg/config/config_type.go +++ b/src/backend/dispatch-k8s-manager/pkg/config/config_type.go @@ -10,6 +10,7 @@ type ConfigYaml struct { BuildLess BuildLess `json:"buildless"` BuildAndPushImage BuildAndPushImage `json:"buildAndPushImage"` ApiServer ApiServer `json:"apiServer"` + Docker Docker `json:"docker"` } type Server struct { @@ -144,3 +145,7 @@ type BuildLess struct { VolumeMount VolumeMount `json:"volumeMount"` Volume Volume `json:"volume"` } + +type Docker struct { + Enable bool `json:"enable"` +} diff --git a/src/backend/dispatch-k8s-manager/pkg/constant/constant.go b/src/backend/dispatch-k8s-manager/pkg/constant/constant.go new file mode 100644 index 00000000000..444b4a13be6 --- /dev/null +++ b/src/backend/dispatch-k8s-manager/pkg/constant/constant.go @@ -0,0 +1,5 @@ +package constant + +const ( + KubernetesManagerDebugEnable = "KUBERNETES_MANAGER_DEBUG_ENABLE" +) diff --git a/src/backend/dispatch-k8s-manager/pkg/docker/docker.go b/src/backend/dispatch-k8s-manager/pkg/docker/docker.go new file mode 100644 index 00000000000..ca4991f3a5a --- /dev/null +++ b/src/backend/dispatch-k8s-manager/pkg/docker/docker.go @@ -0,0 +1,91 @@ +package docker + +import ( + "encoding/base64" + "encoding/json" + "io" + "strings" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/client" + "github.com/pkg/errors" + "golang.org/x/net/context" +) + +var cli *client.Client + +func InitDockerCli() error { + c, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + if err != nil { + return err + } + cli = c + + return nil +} + +func ImageList(ctx context.Context) ([]types.ImageSummary, error) { + images, err := cli.ImageList(ctx, types.ImageListOptions{}) + if err != nil { + return nil, errors.Wrap(err, "list image error") + } + + return images, nil +} + +func ImagePull( + ctx context.Context, + ref string, + username string, + password string, +) (string, error) { + imageName := strings.TrimSpace(ref) + + reader, err := cli.ImagePull(ctx, imageName, types.ImagePullOptions{ + RegistryAuth: generateDockerAuth(username, password), + }) + if err != nil { + return "", errors.Wrap(err, "pull new image error") + } + defer reader.Close() + buf := new(strings.Builder) + _, _ = io.Copy(buf, reader) + + return buf.String(), nil +} + +func ImageInspect(ctx context.Context, imageId string) (*types.ImageInspect, error) { + image, _, err := cli.ImageInspectWithRaw(ctx, imageId) + if err != nil { + return nil, errors.Wrap(err, "image inspect error") + } + + return &image, nil +} + +func ImageRemove(ctx context.Context, imageId string, opts types.ImageRemoveOptions) error { + _, err := cli.ImageRemove(ctx, imageId, opts) + if err != nil { + return err + } + + return nil +} + +// generateDockerAuth 创建拉取docker凭据 +func generateDockerAuth(user, password string) string { + if user == "" || password == "" { + return "" + } + + authConfig := types.AuthConfig{ + Username: user, + Password: password, + } + encodedJSON, err := json.Marshal(authConfig) + if err != nil { + panic(err) + } + + return base64.URLEncoding.EncodeToString(encodedJSON) +} diff --git a/src/backend/dispatch-k8s-manager/pkg/docker/type.go b/src/backend/dispatch-k8s-manager/pkg/docker/type.go new file mode 100644 index 00000000000..c89a0d84d55 --- /dev/null +++ b/src/backend/dispatch-k8s-manager/pkg/docker/type.go @@ -0,0 +1,12 @@ +package docker + +type ImagePullPolicyEnum string + +const ( + ImagePullPolicyAlways ImagePullPolicyEnum = "always" + ImagePullPolicyIfNotPresent ImagePullPolicyEnum = "if-not-present" +) + +func (i ImagePullPolicyEnum) String() string { + return string(i) +} diff --git a/src/backend/dispatch-k8s-manager/pkg/kubeclient/deployment.go b/src/backend/dispatch-k8s-manager/pkg/kubeclient/deployment.go index 6ac30b81abe..804c81dfcde 100644 --- a/src/backend/dispatch-k8s-manager/pkg/kubeclient/deployment.go +++ b/src/backend/dispatch-k8s-manager/pkg/kubeclient/deployment.go @@ -96,6 +96,32 @@ func CreateDeployment(dep *Deployment) error { return nil } +func CreateNativeDeployment(namespace string, deployment *appsv1.Deployment) error { + _, err := kubeClient.AppsV1().Deployments(namespace).Create( + context.TODO(), + deployment, + metav1.CreateOptions{}, + ) + + if err != nil { + return err + } + return nil +} + +func UpdateNativeDeployment(namespace string, deployment *appsv1.Deployment) error { + _, err := kubeClient.AppsV1().Deployments(namespace).Update( + context.TODO(), + deployment, + metav1.UpdateOptions{}, + ) + + if err != nil { + return err + } + return nil +} + func PatchDeployment(deploymentName string, jsonPatch []byte) error { _, err := kubeClient.AppsV1().Deployments(config.Config.Kubernetes.NameSpace).Patch( context.TODO(), @@ -141,3 +167,25 @@ func ListDeployment(workloadCoreLabel string) ([]*appsv1.Deployment, error) { return list, nil } + +func GetNativeDeployment(namespace string, deploymentName string) (*appsv1.Deployment, error) { + deployment, err := kubeClient.AppsV1().Deployments(namespace).Get( + context.TODO(), + deploymentName, + metav1.GetOptions{}, + ) + + if err != nil { + return nil, err + } + + return deployment, nil +} + +func DeleteNativeDeployment(namespace string, deploymentName string) error { + return kubeClient.AppsV1().Deployments(namespace).Delete( + context.TODO(), + deploymentName, + metav1.DeleteOptions{}, + ) +} diff --git a/src/backend/dispatch-k8s-manager/pkg/kubeclient/ingress.go b/src/backend/dispatch-k8s-manager/pkg/kubeclient/ingress.go new file mode 100644 index 00000000000..0668fc223a2 --- /dev/null +++ b/src/backend/dispatch-k8s-manager/pkg/kubeclient/ingress.go @@ -0,0 +1,55 @@ +package kubeclient + +import ( + "context" + networkv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func CreateIngress(namespace string, ingress *networkv1.Ingress) error { + _, err := kubeClient.NetworkingV1().Ingresses(namespace).Create( + context.TODO(), + ingress, + metav1.CreateOptions{}, + ) + + if err != nil { + return err + } + return nil +} + +func UpdateIngress(namespace string, ingress *networkv1.Ingress) error { + _, err := kubeClient.NetworkingV1().Ingresses(namespace).Update( + context.TODO(), + ingress, + metav1.UpdateOptions{}, + ) + + if err != nil { + return err + } + return nil +} + +func DeleteIngress(namespace string, ingressName string) error { + return kubeClient.NetworkingV1().Ingresses(namespace).Delete( + context.TODO(), + ingressName, + metav1.DeleteOptions{}, + ) +} + +func GetIngress(namespace string, ingressName string) (*networkv1.Ingress, error) { + ingress, err := kubeClient.NetworkingV1().Ingresses(namespace).Get( + context.TODO(), + ingressName, + metav1.GetOptions{}, + ) + + if err != nil { + return nil, err + } + + return ingress, err +} diff --git a/src/backend/dispatch-k8s-manager/pkg/kubeclient/secret.go b/src/backend/dispatch-k8s-manager/pkg/kubeclient/secret.go index d38ea5c4e17..0ef18fe7ecc 100644 --- a/src/backend/dispatch-k8s-manager/pkg/kubeclient/secret.go +++ b/src/backend/dispatch-k8s-manager/pkg/kubeclient/secret.go @@ -44,6 +44,17 @@ func CreateDockerRegistry(dockerSecret *DockerSecret) (*corev1.Secret, error) { return s, nil } +func CreateNativeSecret(namespace string, secret *corev1.Secret) error { + _, err := kubeClient.CoreV1(). + Secrets(namespace). + Create(context.TODO(), secret, metav1.CreateOptions{}) + if err != nil { + return err + } + + return nil +} + // DockerConfigJSON represents a local docker auth config file // for pulling images. type DockerConfigJSON struct { @@ -96,6 +107,28 @@ func ListSecret(workloadCoreLabel string) ([]corev1.Secret, error) { return l.Items, nil } +func GetNativeSecret(namespace string, secretName string) (*corev1.Secret, error) { + l, err := kubeClient.CoreV1().Secrets(namespace).Get( + context.TODO(), + secretName, + metav1.GetOptions{}, + ) + if err != nil { + return nil, err + } + return l, nil +} + +func DeleteNativeSecret(namespace string, secretName string) error { + if err := kubeClient.CoreV1(). + Secrets(namespace). + Delete(context.TODO(), secretName, metav1.DeleteOptions{}); err != nil { + return err + } + + return nil +} + func DeleteSecret(secretName string) error { if err := kubeClient.CoreV1(). Secrets(config.Config.Kubernetes.NameSpace). diff --git a/src/backend/dispatch-k8s-manager/pkg/kubeclient/service.go b/src/backend/dispatch-k8s-manager/pkg/kubeclient/service.go new file mode 100644 index 00000000000..37ab318d9e7 --- /dev/null +++ b/src/backend/dispatch-k8s-manager/pkg/kubeclient/service.go @@ -0,0 +1,55 @@ +package kubeclient + +import ( + "context" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func CreateService(namespace string, service *corev1.Service) error { + _, err := kubeClient.CoreV1().Services(namespace).Create( + context.TODO(), + service, + metav1.CreateOptions{}, + ) + + if err != nil { + return err + } + return nil +} + +func UpdateService(namespace string, service *corev1.Service) error { + _, err := kubeClient.CoreV1().Services(namespace).Update( + context.TODO(), + service, + metav1.UpdateOptions{}, + ) + + if err != nil { + return err + } + return nil +} + +func DeleteService(namespace string, serviceName string) error { + return kubeClient.CoreV1().Services(namespace).Delete( + context.TODO(), + serviceName, + metav1.DeleteOptions{}, + ) +} + +func GetService(namespace string, serviceName string) (*corev1.Service, error) { + service, err := kubeClient.CoreV1().Services(namespace).Get( + context.TODO(), + serviceName, + metav1.GetOptions{}, + ) + + if err != nil { + return nil, err + } + + return service, err +} diff --git a/src/backend/dispatch-k8s-manager/pkg/logs/logs.go b/src/backend/dispatch-k8s-manager/pkg/logs/logs.go index 21b03f0c10d..78aa7a1ecfe 100644 --- a/src/backend/dispatch-k8s-manager/pkg/logs/logs.go +++ b/src/backend/dispatch-k8s-manager/pkg/logs/logs.go @@ -5,9 +5,6 @@ import ( "disaptch-k8s-manager/pkg/config" "disaptch-k8s-manager/pkg/types" "fmt" - "github.com/gin-gonic/gin" - "github.com/sirupsen/logrus" - "gopkg.in/natefinch/lumberjack.v2" "net" "net/http" "net/http/httputil" @@ -15,6 +12,10 @@ import ( "path/filepath" "runtime/debug" "strings" + + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + "gopkg.in/natefinch/lumberjack.v2" ) var Logs *logrus.Logger @@ -118,3 +119,11 @@ func Warn(args ...interface{}) { func Error(args ...interface{}) { Logs.Error(args...) } + +func Errorf(format string, args ...interface{}) { + Logs.Errorf(format, args...) +} + +func WithError(err error) *logrus.Entry { + return Logs.WithError(err) +} diff --git a/src/backend/dispatch-k8s-manager/pkg/task/builder_task.go b/src/backend/dispatch-k8s-manager/pkg/task/builder_task.go index e0c8f6ed7bb..469a9d31652 100644 --- a/src/backend/dispatch-k8s-manager/pkg/task/builder_task.go +++ b/src/backend/dispatch-k8s-manager/pkg/task/builder_task.go @@ -9,17 +9,18 @@ import ( "disaptch-k8s-manager/pkg/prometheus" "disaptch-k8s-manager/pkg/types" "fmt" + "time" + "github.com/pkg/errors" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/watch" - "time" ) func DoCreateBuilder(taskId string, dep *kubeclient.Deployment) { _, err := kubeclient.CreateDockerRegistry(dep.Pod.PullImageSecret) if err != nil { - failTask(taskId, errors.Wrap(err, "create builder pull image secret error").Error()) + FailTask(taskId, errors.Wrap(err, "create builder pull image secret error").Error()) return } @@ -29,14 +30,14 @@ func DoCreateBuilder(taskId string, dep *kubeclient.Deployment) { } // 创建失败后的操作 - failTask(taskId, errors.Wrap(err, "create builder error").Error()) + FailTask(taskId, errors.Wrap(err, "create builder error").Error()) deleteBuilderLinkRes(dep.Name) } func DoStartBuilder(taskId string, builderName string, data []byte) { err := kubeclient.PatchDeployment(builderName, data) if err != nil { - failTask(taskId, errors.Wrap(err, "start builder error").Error()) + FailTask(taskId, errors.Wrap(err, "start builder error").Error()) return } } @@ -55,7 +56,7 @@ func DoStopBuilder(taskId string, builderName string, data []byte) { err = kubeclient.PatchDeployment(builderName, data) if err != nil { - failTask(taskId, errors.Wrap(err, "stop builder error").Error()) + FailTask(taskId, errors.Wrap(err, "stop builder error").Error()) return } @@ -123,14 +124,14 @@ func saveRealResourceUsage(builderName string, pods []*corev1.Pod) error { func DoDeleteBuilder(taskId string, builderName string) { err := kubeclient.DeleteDeployment(builderName) if err != nil { - failTask(taskId, errors.Wrap(err, "delete builder error").Error()) + FailTask(taskId, errors.Wrap(err, "delete builder error").Error()) return } deleteBuilderLinkRes(builderName) deleteBuilderLinkDbData(builderName) - okTask(taskId) + OkTask(taskId) } // deleteBuilderLinkRes 删除构建机相关联的kubernetes资源 @@ -177,7 +178,7 @@ func watchBuilderTaskPodCreateOrStart(event watch.Event, pod *corev1.Pod, taskId { switch podStatus.Phase { case corev1.PodPending: - updateTask(taskId, types.TaskRunning) + UpdateTask(taskId, types.TaskRunning) // 对于task的start/create来说，启动了就算成功，而不关心启动成功还是失败了 case corev1.PodRunning, corev1.PodSucceeded, corev1.PodFailed: { @@ -206,7 +207,7 @@ func watchBuilderTaskPodCreateOrStart(event watch.Event, pod *corev1.Pod, taskId } defer redis.UnLock(key) - okTask(taskId) + OkTask(taskId) // mysql中保存分配至节点成功的构建机最近三次节点信息，用来做下一次调度的依据 if builderName == "" { @@ -222,13 +223,13 @@ func watchBuilderTaskPodCreateOrStart(event watch.Event, pod *corev1.Pod, taskId return } case corev1.PodUnknown: - updateTask(taskId, types.TaskUnknown) + UpdateTask(taskId, types.TaskUnknown) } } case watch.Error: { logs.Error("add job error. ", pod) - failTask(taskId, podStatus.Message+"|"+podStatus.Reason) + FailTask(taskId, podStatus.Message+"|"+podStatus.Reason) } } } @@ -265,14 +266,14 @@ func watchBuilderTaskDeploymentStop(event watch.Event, dep *appsv1.Deployment, t switch event.Type { case watch.Modified: if dep.Spec.Replicas != nil && *dep.Spec.Replicas == 0 { - okTask(taskId) + OkTask(taskId) } case watch.Error: logs.Error("stop builder error. ", dep) if len(dep.Status.Conditions) > 0 { - failTask(taskId, dep.Status.Conditions[0].String()) + FailTask(taskId, dep.Status.Conditions[0].String()) } else { - failTask(taskId, "stop builder error") + FailTask(taskId, "stop builder error") } } } diff --git a/src/backend/dispatch-k8s-manager/pkg/task/job_task.go b/src/backend/dispatch-k8s-manager/pkg/task/job_task.go index 0c8abc352c4..478363cdd98 100644 --- a/src/backend/dispatch-k8s-manager/pkg/task/job_task.go +++ b/src/backend/dispatch-k8s-manager/pkg/task/job_task.go @@ -5,6 +5,7 @@ import ( "disaptch-k8s-manager/pkg/logs" "disaptch-k8s-manager/pkg/types" "fmt" + "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/watch" @@ -18,12 +19,12 @@ func DoCreateBuildAndPushImageJob( // 创建镜像拉取凭据 _, err := kubeclient.CreateDockerRegistry(job.Pod.PullImageSecret) if err != nil { - failTask(taskId, errors.Wrap(err, "create build and push image job pull image secret error").Error()) + FailTask(taskId, errors.Wrap(err, "create build and push image job pull image secret error").Error()) return } if _, err = kubeclient.CreateDockerRegistry(kanikoSecret); err != nil { - failTask(taskId, errors.Wrap(err, "create build and push image push secret error").Error()) + FailTask(taskId, errors.Wrap(err, "create build and push image push secret error").Error()) return } @@ -32,14 +33,14 @@ func DoCreateBuildAndPushImageJob( return } - failTask(taskId, errors.Wrap(err, "create job error").Error()) + FailTask(taskId, errors.Wrap(err, "create job error").Error()) deleteJobLinkRes(job.Name) } func DoCreateJob(taskId string, job *kubeclient.Job) { _, err := kubeclient.CreateDockerRegistry(job.Pod.PullImageSecret) if err != nil { - failTask(taskId, errors.Wrap(err, "create job pull image secret error").Error()) + FailTask(taskId, errors.Wrap(err, "create job pull image secret error").Error()) return } @@ -49,7 +50,7 @@ func DoCreateJob(taskId string, job *kubeclient.Job) { } // 创建失败后的操作 - failTask(taskId, errors.Wrap(err, "create job error").Error()) + FailTask(taskId, errors.Wrap(err, "create job error").Error()) deleteJobLinkRes(job.Name) } @@ -57,13 +58,13 @@ func DoCreateJob(taskId string, job *kubeclient.Job) { func DoDeleteJob(taskId string, jobName string) { err := kubeclient.DeleteJob(jobName) if err != nil { - failTask(taskId, errors.Wrap(err, "delete job error").Error()) + FailTask(taskId, errors.Wrap(err, "delete job error").Error()) return } deleteJobLinkRes(jobName) - okTask(taskId) + OkTask(taskId) } // deleteJobLinkRes 删除JOB相关联的kubernetes资源 @@ -103,18 +104,18 @@ func watchJobTaskPodCreateOrStart(event watch.Event, pod *corev1.Pod, taskId str { switch podStatus.Phase { case corev1.PodPending: - updateTask(taskId, types.TaskRunning) + UpdateTask(taskId, types.TaskRunning) // 对于task的start/create来说，启动了就算成功，而不关系启动成功还是失败了 case corev1.PodRunning, corev1.PodSucceeded, corev1.PodFailed: - okTask(taskId) + OkTask(taskId) case corev1.PodUnknown: - updateTask(taskId, types.TaskUnknown) + UpdateTask(taskId, types.TaskUnknown) } } case watch.Error: { logs.Error("add job error. ", pod) - failTask(taskId, podStatus.Message+"|"+podStatus.Reason) + FailTask(taskId, podStatus.Message+"|"+podStatus.Reason) } } } diff --git a/src/backend/dispatch-k8s-manager/pkg/task/task.go b/src/backend/dispatch-k8s-manager/pkg/task/task.go index dc9fbf4c388..bb9bbf4fbff 100644 --- a/src/backend/dispatch-k8s-manager/pkg/task/task.go +++ b/src/backend/dispatch-k8s-manager/pkg/task/task.go @@ -4,7 +4,6 @@ import ( "disaptch-k8s-manager/pkg/db/mysql" "disaptch-k8s-manager/pkg/logs" "disaptch-k8s-manager/pkg/types" - "github.com/pkg/errors" ) func InitTask() { @@ -12,23 +11,30 @@ func InitTask() { go WatchTaskDeployment() } -func okTask(taskId string) { +func OkTask(taskId string) { err := mysql.UpdateTask(taskId, types.TaskSucceeded, "") if err != nil { - logs.Error(errors.Wrapf(err, "save okTask %s %s error. ", taskId, "")) + logs.Errorf("save OkTask %s error %s", taskId, err.Error()) } } -func updateTask(taskId string, state types.TaskState) { +func OkTaskWithMessage(taskId string, message string) { + err := mysql.UpdateTask(taskId, types.TaskSucceeded, message) + if err != nil { + logs.Errorf("save OkTaskWithMessage %s %s error %s", taskId, message, err.Error()) + } +} + +func UpdateTask(taskId string, state types.TaskState) { err := mysql.UpdateTask(taskId, state, "") if err != nil { - logs.Error(errors.Wrapf(err, "update okTask %s %s error. ", taskId, "")) + logs.Errorf("save UpdateTask %s %s error %s", taskId, state, err.Error()) } } -func failTask(taskId string, message string) { +func FailTask(taskId string, message string) { err := mysql.UpdateTask(taskId, types.TaskFailed, message) if err != nil { - logs.Error(errors.Wrapf(err, "save failTask %s %s error. ", taskId, message)) + logs.Errorf("save FailTask %s %s error %s", taskId, message, err.Error()) } } diff --git a/src/backend/dispatch-k8s-manager/pkg/types/task_type.go b/src/backend/dispatch-k8s-manager/pkg/types/task_type.go index ce8bdb9bdd9..75f5bc54043 100644 --- a/src/backend/dispatch-k8s-manager/pkg/types/task_type.go +++ b/src/backend/dispatch-k8s-manager/pkg/types/task_type.go @@ -23,6 +23,11 @@ const ( TaskActionDelete TaskAction = "delete" ) +// docker 交互操作 +const ( + TaskDockerActionInspect TaskAction = "inspect" +) + type TaskLabelType string const ( @@ -36,6 +41,7 @@ type TaskBelong string const ( TaskBelongBuilder = "builder" TaskBelongJob = "job" + TaskBelongDocker = "docker" ) type Task struct { diff --git a/src/backend/dispatch-k8s-manager/resources/config.yaml b/src/backend/dispatch-k8s-manager/resources/config.yaml index fd6e7c6ad3c..0a923f22ab4 100644 --- a/src/backend/dispatch-k8s-manager/resources/config.yaml +++ b/src/backend/dispatch-k8s-manager/resources/config.yaml @@ -2,7 +2,7 @@ server: port: 8081 mysql: - dataSourceName: root:123456@tcp(localhost:3306)/devops_kubernetes_manager?parseTime=true&loc=Local + dataSourceName: root:123456@tcp(localhost:3306)/devops_ci_kubernetes_manager?parseTime=true&loc=Local connMaxLifetime: 3 maxOpenConns: 10 maxIdleConns: 10 @@ -108,6 +108,9 @@ apiserver: apiToken: key: Devops-Token value: landun - rsaPrivateKey: | - # 在这里保存私钥用来解密apitoken - # 推荐使用rsa-generate生成公私钥，rsa-generate可通过make打包获得 + # 在这里保存私钥用来解密apitoken + # 推荐使用rsa-generate生成公私钥，rsa-generate可通过make打包获得 + rsaPrivateKey: "" + +docker: + enable: true diff --git a/src/backend/dispatch-k8s-manager/swagger/apiserver/docs.go b/src/backend/dispatch-k8s-manager/swagger/apiserver/docs.go index 609c9e87d6b..b1a5352f9bf 100644 --- a/src/backend/dispatch-k8s-manager/swagger/apiserver/docs.go +++ b/src/backend/dispatch-k8s-manager/swagger/apiserver/docs.go @@ -1,5 +1,5 @@ -// Package apiserver GENERATED BY SWAG; DO NOT EDIT -// This file was generated by swaggo/swag +// Code generated by swaggo/swag. DO NOT EDIT. + package apiserver import "github.com/swaggo/swag" @@ -309,6 +309,55 @@ const docTemplate = `{ } } }, + "/docker/inspect": { + "post": { + "consumes": [ + "application/json" + ], + "tags": [ + "docker" + ], + "summary": "docker inspect命令(同时会pull)", + "parameters": [ + { + "type": "string", + "description": "凭证信息", + "name": "Devops-Token", + "in": "header", + "required": true + }, + { + "description": "构建机信息", + "name": "info", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/service.DockerInspectInfo" + } + } + ], + "responses": { + "200": { + "description": "任务ID", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/types.Result" + }, + { + "type": "object", + "properties": { + "data": { + "$ref": "#/definitions/service.TaskId" + } + } + } + ] + } + } + } + } + }, "/jobs": { "post": { "consumes": [ @@ -618,7 +667,11 @@ const docTemplate = `{ }, "info": { "description": "构建并推送镜像的具体信息", - "$ref": "#/definitions/service.buildImageInfo" + "allOf": [ + { + "$ref": "#/definitions/service.buildImageInfo" + } + ] }, "name": { "description": "唯一名称", @@ -627,11 +680,19 @@ const docTemplate = `{ }, "podNameSelector": { "description": "Pod名称调度", - "$ref": "#/definitions/service.PodNameSelector" + "allOf": [ + { + "$ref": "#/definitions/service.PodNameSelector" + } + ] }, "resource": { "description": "工作负载资源", - "$ref": "#/definitions/service.CommonWorkLoadResource" + "allOf": [ + { + "$ref": "#/definitions/service.CommonWorkLoadResource" + } + ] } } }, @@ -675,19 +736,35 @@ const docTemplate = `{ }, "privateBuilder": { "description": "私有构建机配置", - "$ref": "#/definitions/service.DedicatedBuilder" + "allOf": [ + { + "$ref": "#/definitions/service.DedicatedBuilder" + } + ] }, "registry": { "description": "镜像凭证", - "$ref": "#/definitions/types.Registry" + "allOf": [ + { + "$ref": "#/definitions/types.Registry" + } + ] }, "resource": { "description": "工作负载资源", - "$ref": "#/definitions/service.CommonWorkLoadResource" + "allOf": [ + { + "$ref": "#/definitions/service.CommonWorkLoadResource" + } + ] }, "specialBuilder": { "description": "特殊构建机配置", - "$ref": "#/definitions/service.DedicatedBuilder" + "allOf": [ + { + "$ref": "#/definitions/service.DedicatedBuilder" + } + ] } } }, @@ -708,6 +785,27 @@ const docTemplate = `{ } } }, + "service.BuilderState": { + "type": "string", + "enum": [ + "readyToRun", + "notExist", + "pending", + "running", + "succeeded", + "failed", + "unknown" + ], + "x-enum-varnames": [ + "BuilderReadyToRun", + "BuilderNotExist", + "BuilderPending", + "BuilderRunning", + "BuilderSucceeded", + "BuilderFailed", + "BuilderUnknown" + ] + }, "service.BuilderStatus": { "type": "object", "properties": { @@ -715,7 +813,7 @@ const docTemplate = `{ "type": "string" }, "status": { - "type": "string" + "$ref": "#/definitions/service.BuilderState" } } }, @@ -766,6 +864,17 @@ const docTemplate = `{ } } }, + "service.Credential": { + "type": "object", + "properties": { + "password": { + "type": "string" + }, + "username": { + "type": "string" + } + } + }, "service.DedicatedBuilder": { "type": "object", "properties": { @@ -774,6 +883,31 @@ const docTemplate = `{ } } }, + "service.DockerInspectInfo": { + "type": "object", + "required": [ + "name", + "ref" + ], + "properties": { + "cred": { + "description": "拉取镜像凭据", + "allOf": [ + { + "$ref": "#/definitions/service.Credential" + } + ] + }, + "name": { + "description": "任务名称，唯一", + "type": "string" + }, + "ref": { + "description": "docker镜像信息 如：docker:latest", + "type": "string" + } + } + }, "service.Job": { "type": "object", "required": [ @@ -818,18 +952,47 @@ const docTemplate = `{ }, "podNameSelector": { "description": "Pod名称调度选项", - "$ref": "#/definitions/service.PodNameSelector" + "allOf": [ + { + "$ref": "#/definitions/service.PodNameSelector" + } + ] }, "registry": { "description": "镜像凭证", - "$ref": "#/definitions/types.Registry" + "allOf": [ + { + "$ref": "#/definitions/types.Registry" + } + ] }, "resource": { "description": "工作负载资源", - "$ref": "#/definitions/service.CommonWorkLoadResource" + "allOf": [ + { + "$ref": "#/definitions/service.CommonWorkLoadResource" + } + ] } } }, + "service.JobState": { + "type": "string", + "enum": [ + "pending", + "running", + "succeeded", + "failed", + "unknown" + ], + "x-enum-varnames": [ + "JobPending", + "JobRunning", + "JobSucceeded", + "JobFailed", + "JobUnknown" + ] + }, "service.JobStatus": { "type": "object", "properties": { @@ -840,7 +1003,7 @@ const docTemplate = `{ "type": "string" }, "state": { - "type": "string" + "$ref": "#/definitions/service.JobState" } } }, @@ -875,7 +1038,7 @@ const docTemplate = `{ "type": "string" }, "status": { - "type": "string" + "$ref": "#/definitions/types.TaskState" } } }, @@ -958,6 +1121,23 @@ const docTemplate = `{ "type": "integer" } } + }, + "types.TaskState": { + "type": "string", + "enum": [ + "waiting", + "running", + "succeeded", + "failed", + "unknown" + ], + "x-enum-varnames": [ + "TaskWaiting", + "TaskRunning", + "TaskSucceeded", + "TaskFailed", + "TaskUnknown" + ] } } }` @@ -972,6 +1152,8 @@ var SwaggerInfo = &swag.Spec{ Description: "", InfoInstanceName: "swagger", SwaggerTemplate: docTemplate, + LeftDelim: "{{", + RightDelim: "}}", } func init() { diff --git a/src/backend/dispatch-k8s-manager/swagger/apiserver/swagger.json b/src/backend/dispatch-k8s-manager/swagger/apiserver/swagger.json index d53e59cc2a0..66563b9aa23 100644 --- a/src/backend/dispatch-k8s-manager/swagger/apiserver/swagger.json +++ b/src/backend/dispatch-k8s-manager/swagger/apiserver/swagger.json @@ -300,6 +300,55 @@ } } }, + "/docker/inspect": { + "post": { + "consumes": [ + "application/json" + ], + "tags": [ + "docker" + ], + "summary": "docker inspect命令(同时会pull)", + "parameters": [ + { + "type": "string", + "description": "凭证信息", + "name": "Devops-Token", + "in": "header", + "required": true + }, + { + "description": "构建机信息", + "name": "info", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/service.DockerInspectInfo" + } + } + ], + "responses": { + "200": { + "description": "任务ID", + "schema": { + "allOf": [ + { + "$ref": "#/definitions/types.Result" + }, + { + "type": "object", + "properties": { + "data": { + "$ref": "#/definitions/service.TaskId" + } + } + } + ] + } + } + } + } + }, "/jobs": { "post": { "consumes": [ @@ -609,7 +658,11 @@ }, "info": { "description": "构建并推送镜像的具体信息", - "$ref": "#/definitions/service.buildImageInfo" + "allOf": [ + { + "$ref": "#/definitions/service.buildImageInfo" + } + ] }, "name": { "description": "唯一名称", @@ -618,11 +671,19 @@ }, "podNameSelector": { "description": "Pod名称调度", - "$ref": "#/definitions/service.PodNameSelector" + "allOf": [ + { + "$ref": "#/definitions/service.PodNameSelector" + } + ] }, "resource": { "description": "工作负载资源", - "$ref": "#/definitions/service.CommonWorkLoadResource" + "allOf": [ + { + "$ref": "#/definitions/service.CommonWorkLoadResource" + } + ] } } }, @@ -666,19 +727,35 @@ }, "privateBuilder": { "description": "私有构建机配置", - "$ref": "#/definitions/service.DedicatedBuilder" + "allOf": [ + { + "$ref": "#/definitions/service.DedicatedBuilder" + } + ] }, "registry": { "description": "镜像凭证", - "$ref": "#/definitions/types.Registry" + "allOf": [ + { + "$ref": "#/definitions/types.Registry" + } + ] }, "resource": { "description": "工作负载资源", - "$ref": "#/definitions/service.CommonWorkLoadResource" + "allOf": [ + { + "$ref": "#/definitions/service.CommonWorkLoadResource" + } + ] }, "specialBuilder": { "description": "特殊构建机配置", - "$ref": "#/definitions/service.DedicatedBuilder" + "allOf": [ + { + "$ref": "#/definitions/service.DedicatedBuilder" + } + ] } } }, @@ -699,6 +776,27 @@ } } }, + "service.BuilderState": { + "type": "string", + "enum": [ + "readyToRun", + "notExist", + "pending", + "running", + "succeeded", + "failed", + "unknown" + ], + "x-enum-varnames": [ + "BuilderReadyToRun", + "BuilderNotExist", + "BuilderPending", + "BuilderRunning", + "BuilderSucceeded", + "BuilderFailed", + "BuilderUnknown" + ] + }, "service.BuilderStatus": { "type": "object", "properties": { @@ -706,7 +804,7 @@ "type": "string" }, "status": { - "type": "string" + "$ref": "#/definitions/service.BuilderState" } } }, @@ -757,6 +855,17 @@ } } }, + "service.Credential": { + "type": "object", + "properties": { + "password": { + "type": "string" + }, + "username": { + "type": "string" + } + } + }, "service.DedicatedBuilder": { "type": "object", "properties": { @@ -765,6 +874,31 @@ } } }, + "service.DockerInspectInfo": { + "type": "object", + "required": [ + "name", + "ref" + ], + "properties": { + "cred": { + "description": "拉取镜像凭据", + "allOf": [ + { + "$ref": "#/definitions/service.Credential" + } + ] + }, + "name": { + "description": "任务名称，唯一", + "type": "string" + }, + "ref": { + "description": "docker镜像信息 如：docker:latest", + "type": "string" + } + } + }, "service.Job": { "type": "object", "required": [ @@ -809,18 +943,47 @@ }, "podNameSelector": { "description": "Pod名称调度选项", - "$ref": "#/definitions/service.PodNameSelector" + "allOf": [ + { + "$ref": "#/definitions/service.PodNameSelector" + } + ] }, "registry": { "description": "镜像凭证", - "$ref": "#/definitions/types.Registry" + "allOf": [ + { + "$ref": "#/definitions/types.Registry" + } + ] }, "resource": { "description": "工作负载资源", - "$ref": "#/definitions/service.CommonWorkLoadResource" + "allOf": [ + { + "$ref": "#/definitions/service.CommonWorkLoadResource" + } + ] } } }, + "service.JobState": { + "type": "string", + "enum": [ + "pending", + "running", + "succeeded", + "failed", + "unknown" + ], + "x-enum-varnames": [ + "JobPending", + "JobRunning", + "JobSucceeded", + "JobFailed", + "JobUnknown" + ] + }, "service.JobStatus": { "type": "object", "properties": { @@ -831,7 +994,7 @@ "type": "string" }, "state": { - "type": "string" + "$ref": "#/definitions/service.JobState" } } }, @@ -866,7 +1029,7 @@ "type": "string" }, "status": { - "type": "string" + "$ref": "#/definitions/types.TaskState" } } }, @@ -949,6 +1112,23 @@ "type": "integer" } } + }, + "types.TaskState": { + "type": "string", + "enum": [ + "waiting", + "running", + "succeeded", + "failed", + "unknown" + ], + "x-enum-varnames": [ + "TaskWaiting", + "TaskRunning", + "TaskSucceeded", + "TaskFailed", + "TaskUnknown" + ] } } } \ No newline at end of file diff --git a/src/backend/dispatch-k8s-manager/swagger/apiserver/swagger.yaml b/src/backend/dispatch-k8s-manager/swagger/apiserver/swagger.yaml index 03fecc91edb..96ebce4efc4 100644 --- a/src/backend/dispatch-k8s-manager/swagger/apiserver/swagger.yaml +++ b/src/backend/dispatch-k8s-manager/swagger/apiserver/swagger.yaml @@ -6,17 +6,20 @@ definitions: description: Job存活时间 type: integer info: - $ref: '#/definitions/service.buildImageInfo' + allOf: + - $ref: '#/definitions/service.buildImageInfo' description: 构建并推送镜像的具体信息 name: description: 唯一名称 maxLength: 32 type: string podNameSelector: - $ref: '#/definitions/service.PodNameSelector' + allOf: + - $ref: '#/definitions/service.PodNameSelector' description: Pod名称调度 resource: - $ref: '#/definitions/service.CommonWorkLoadResource' + allOf: + - $ref: '#/definitions/service.CommonWorkLoadResource' description: 工作负载资源 required: - info @@ -49,16 +52,20 @@ definitions: $ref: '#/definitions/types.NFS' type: array privateBuilder: - $ref: '#/definitions/service.DedicatedBuilder' + allOf: + - $ref: '#/definitions/service.DedicatedBuilder' description: 私有构建机配置 registry: - $ref: '#/definitions/types.Registry' + allOf: + - $ref: '#/definitions/types.Registry' description: 镜像凭证 resource: - $ref: '#/definitions/service.CommonWorkLoadResource' + allOf: + - $ref: '#/definitions/service.CommonWorkLoadResource' description: 工作负载资源 specialBuilder: - $ref: '#/definitions/service.DedicatedBuilder' + allOf: + - $ref: '#/definitions/service.DedicatedBuilder' description: 特殊构建机配置 required: - image @@ -76,12 +83,30 @@ definitions: type: string type: object type: object + service.BuilderState: + enum: + - readyToRun + - notExist + - pending + - running + - succeeded + - failed + - unknown + type: string + x-enum-varnames: + - BuilderReadyToRun + - BuilderNotExist + - BuilderPending + - BuilderRunning + - BuilderSucceeded + - BuilderFailed + - BuilderUnknown service.BuilderStatus: properties: message: type: string status: - type: string + $ref: '#/definitions/service.BuilderState' type: object service.CommonWorkLoadResource: properties: @@ -119,11 +144,34 @@ definitions: - requestDiskIO - requestMem type: object + service.Credential: + properties: + password: + type: string + username: + type: string + type: object service.DedicatedBuilder: properties: name: type: string type: object + service.DockerInspectInfo: + properties: + cred: + allOf: + - $ref: '#/definitions/service.Credential' + description: 拉取镜像凭据 + name: + description: 任务名称，唯一 + type: string + ref: + description: docker镜像信息 如：docker:latest + type: string + required: + - name + - ref + type: object service.Job: properties: activeDeadlineSeconds: @@ -152,19 +200,36 @@ definitions: $ref: '#/definitions/types.NFS' type: array podNameSelector: - $ref: '#/definitions/service.PodNameSelector' + allOf: + - $ref: '#/definitions/service.PodNameSelector' description: Pod名称调度选项 registry: - $ref: '#/definitions/types.Registry' + allOf: + - $ref: '#/definitions/types.Registry' description: 镜像凭证 resource: - $ref: '#/definitions/service.CommonWorkLoadResource' + allOf: + - $ref: '#/definitions/service.CommonWorkLoadResource' description: 工作负载资源 required: - image - name - resource type: object + service.JobState: + enum: + - pending + - running + - succeeded + - failed + - unknown + type: string + x-enum-varnames: + - JobPending + - JobRunning + - JobSucceeded + - JobFailed + - JobUnknown service.JobStatus: properties: message: @@ -172,7 +237,7 @@ definitions: podIp: type: string state: - type: string + $ref: '#/definitions/service.JobState' type: object service.PodNameSelector: properties: @@ -195,7 +260,7 @@ definitions: detail: type: string status: - type: string + $ref: '#/definitions/types.TaskState' type: object service.buildImageInfo: properties: @@ -252,6 +317,20 @@ definitions: status: type: integer type: object + types.TaskState: + enum: + - waiting + - running + - succeeded + - failed + - unknown + type: string + x-enum-varnames: + - TaskWaiting + - TaskRunning + - TaskSucceeded + - TaskFailed + - TaskUnknown info: contact: {} title: kubernetes-manager api文档 @@ -432,6 +511,35 @@ paths: summary: 获取远程登录链接 tags: - builder + /docker/inspect: + post: + consumes: + - application/json + parameters: + - description: 凭证信息 + in: header + name: Devops-Token + required: true + type: string + - description: 构建机信息 + in: body + name: info + required: true + schema: + $ref: '#/definitions/service.DockerInspectInfo' + responses: + "200": + description: 任务ID + schema: + allOf: + - $ref: '#/definitions/types.Result' + - properties: + data: + $ref: '#/definitions/service.TaskId' + type: object + summary: docker inspect命令(同时会pull) + tags: + - docker /jobs: post: consumes: diff --git a/src/backend/dispatch-k8s-manager/swagger/init-swager.sh b/src/backend/dispatch-k8s-manager/swagger/init-swager.sh old mode 100644 new mode 100755 diff --git a/src/frontend/bk-pipeline/dist/bk-pipeline.min.js b/src/frontend/bk-pipeline/dist/bk-pipeline.min.js index 15df0332388..5d8c8f05764 100644 --- a/src/frontend/bk-pipeline/dist/bk-pipeline.min.js +++ b/src/frontend/bk-pipeline/dist/bk-pipeline.min.js @@ -1,2 +1,2 @@ /*! For license information please see bk-pipeline.min.js.LICENSE.txt */ -!function(e,t){"object"==typeof exports&&"object"==typeof module?module.exports=t(require("vue"),require("bk-magic-vue")):"function"==typeof define&&define.amd?define(["vue","bk-magic-vue"],t):"object"==typeof exports?exports.bkPipeline=t(require("vue"),require("bk-magic-vue")):e.bkPipeline=t(e.Vue,e.bkMagic)}(self,((e,t)=>(()=>{var i={606:()=>{!function(){const e='';document.body?document.body.insertAdjacentHTML("afterbegin",e):document.addEventListener("DOMContentLoaded",(function(){document.body.insertAdjacentHTML("afterbegin",e)}))}()},688:(e,t,i)=>{"use strict";i.d(t,{A:()=>s});var n=i(441),o=i.n(n),a=i(562),r=i.n(a)()(o());r.push([e.id,'/*!\n * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.\n *\n * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.\n *\n * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.\n *\n * A copy of the MIT License is included in this file.\n *\n *\n * Terms of the MIT License:\n * ---------------------------------------------------\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n *//*!\n * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.\n *\n * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.\n *\n * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.\n *\n * A copy of the MIT License is included in this file.\n *\n *\n * Terms of the MIT License:\n * ---------------------------------------------------\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */@keyframes rotating{from{transform:rotate(0)}to{transform:rotate(359deg)}}.connect-line{position:absolute;width:58px;top:19px;stroke:#3c96ff;stroke-width:1;fill:none}.insert-tip{position:absolute;display:block;padding:0 10px;max-width:100px;height:24px;display:flex;align-items:center;border:1px solid #addaff;border-radius:22px;color:#3c96ff;font-size:10px;cursor:pointer;background-color:#fff;box-shadow:0px 2px 4px 0px rgba(60,150,255,.2)}.insert-tip.direction:after{content:"";position:absolute;height:6px;width:6px;background-color:#fff;transform:rotate(45deg);bottom:-4px;left:20px;border-right:1px solid #addaff;border-bottom:1px solid #addaff}.insert-tip .tip-icon{margin:0 5px 0 0;cursor:pointer;position:relative;display:block;width:8px;height:8px;transition:all .3s ease}.insert-tip .tip-icon:before,.insert-tip .tip-icon:after{content:"";position:absolute;left:3px;top:0px;height:8px;width:2px;background-color:#3c96ff}.insert-tip .tip-icon:after{transform:rotate(90deg)}.insert-tip>span{white-space:nowrap}.insert-tip:hover{background-color:#3c96ff;color:#fff;border-color:#3c96ff}.insert-tip:hover.direction:after{background-color:#3c96ff;border-right-color:#3c96ff;border-bottom-color:#3c96ff}.insert-tip:hover .tip-icon{position:relative;display:block;width:8px;height:8px;transition:all .3s ease}.insert-tip:hover .tip-icon:before,.insert-tip:hover .tip-icon:after{content:"";position:absolute;left:3px;top:0px;height:8px;width:2px;background-color:#fff}.insert-tip:hover .tip-icon:after{transform:rotate(90deg)}.pointer{cursor:pointer}span.skip-name{text-decoration:line-through;color:#c4cdd6}span.skip-name:hover{color:#c4cdd6}.spin-icon{display:inline-block;animation:loading .8s linear infinite}.add-plus-icon,.minus-icon{position:relative;display:block;width:18px;height:18px;border:1px solid #addaff;background-color:#fff;border-radius:50%;transition:all .3s ease}.add-plus-icon:before,.minus-icon:before,.add-plus-icon:after,.minus-icon:after{content:"";position:absolute;left:8px;top:5px;left:7px;top:4px;height:8px;width:2px;background-color:#3c96ff}.add-plus-icon:after,.minus-icon:after{transform:rotate(90deg)}.add-plus-icon.active,.active.minus-icon{border-color:#3c96ff;background-color:#3c96ff}.add-plus-icon.active:before,.active.minus-icon:before,.add-plus-icon.active:after,.active.minus-icon:after{background-color:#fff}.add-plus-icon:hover,.minus-icon:hover{border-color:#3c96ff;background-color:#3c96ff}.add-plus-icon:hover:before,.minus-icon:hover:before,.add-plus-icon:hover:after,.minus-icon:hover:after{background-color:#fff}.minus-icon:before{display:none}.un-exec-this-time{opacity:.5}.un-exec-this-time .un-exec-this-time{opacity:1}.readonly .stage-connector{background:#c3cdd7;color:#c3cdd7}.readonly .stage-connector:before{background:#c3cdd7}.readonly .connect-line.left,.readonly .connect-line.right{stroke:#c3cdd7}.readonly .connect-line.left:before,.readonly .connect-line.right:before{stroke:#c3cdd7}.readonly .connect-line.left:after,.readonly .connect-line.right:after{stroke:#c3cdd7;background-color:#c3cdd7}.readonly:after{background:#c3cdd7}.readonly .container-connect-triangle{color:#c3cdd7}.readonly .container-title{cursor:pointer;background-color:#63656e}.readonly .container-title:before,.readonly .container-title:after{border-top-color:#c3cdd7}.readonly .container-title>.container-name span{color:#fff}.editing .stage-connector{background:#3c96ff;color:#3c96ff}.editing .stage-connector:before{background:#3c96ff}.editing .connect-line.left,.editing .connect-line.right{stroke:#3c96ff}.editing .connect-line.left:before,.editing .connect-line.right:before{stroke:#3c96ff}.editing .connect-line.left:after,.editing .connect-line.right:after{stroke:#3c96ff;background-color:#3c96ff}.editing:after{background:#3c96ff}.editing:before{color:#3c96ff}.container-type{font-size:12px;margin-right:12px;font-style:normal}.container-type .devops-icon{font-size:18px}.container-type .devops-icon.icon-exclamation-triangle-shape{font-size:14px}.container-type .devops-icon.icon-exclamation-triangle-shape.is-danger{color:#ff5656}.container-type i{font-style:normal}.bk-pipeline .stage-status:not(.readonly){background-color:#3c96ff}.bk-pipeline .stage-status:not(.readonly).WARNING{background-color:#ffb400;color:#fff}.bk-pipeline .stage-status:not(.readonly).FAILED{background-color:#ff5656;color:#fff}.bk-pipeline .stage-status:not(.readonly).SUCCEED{background-color:#34d97b;color:#fff}.bk-pipeline .stage-status.CANCELED,.bk-pipeline .stage-status.REVIEW_ABORT,.bk-pipeline .stage-status.REVIEWING,.bk-pipeline .stage-name-status-icon.CANCELED,.bk-pipeline .stage-name-status-icon.REVIEW_ABORT,.bk-pipeline .stage-name-status-icon.REVIEWING{color:#ffb400}.bk-pipeline .stage-status.FAILED,.bk-pipeline .stage-status.QUALITY_CHECK_FAIL,.bk-pipeline .stage-status.HEARTBEAT_TIMEOUT,.bk-pipeline .stage-status.QUEUE_TIMEOUT,.bk-pipeline .stage-status.EXEC_TIMEOUT,.bk-pipeline .stage-name-status-icon.FAILED,.bk-pipeline .stage-name-status-icon.QUALITY_CHECK_FAIL,.bk-pipeline .stage-name-status-icon.HEARTBEAT_TIMEOUT,.bk-pipeline .stage-name-status-icon.QUEUE_TIMEOUT,.bk-pipeline .stage-name-status-icon.EXEC_TIMEOUT{color:#ff5656}.bk-pipeline .stage-status.SUCCEED,.bk-pipeline .stage-status.REVIEW_PROCESSED,.bk-pipeline .stage-name-status-icon.SUCCEED,.bk-pipeline .stage-name-status-icon.REVIEW_PROCESSED{color:#34d97b}.bk-pipeline .stage-status.PAUSE,.bk-pipeline .stage-name-status-icon.PAUSE{color:#63656e}.bk-pipeline .container-title .stage-status{color:#fff}.bk-pipeline .container-title.UNEXEC,.bk-pipeline .container-title.SKIP,.bk-pipeline .container-title.DISABLED{color:#fff;background-color:#63656e}.bk-pipeline .container-title.UNEXEC .fold-atom-icon,.bk-pipeline .container-title.SKIP .fold-atom-icon,.bk-pipeline .container-title.DISABLED .fold-atom-icon{color:#63656e}.bk-pipeline .container-title.QUEUE,.bk-pipeline .container-title.RUNNING,.bk-pipeline .container-title.REVIEWING,.bk-pipeline .container-title.PREPARE_ENV,.bk-pipeline .container-title.LOOP_WAITING,.bk-pipeline .container-title.DEPENDENT_WAITING,.bk-pipeline .container-title.CALL_WAITING{background-color:#459fff;color:#fff}.bk-pipeline .container-title.QUEUE .fold-atom-icon,.bk-pipeline .container-title.RUNNING .fold-atom-icon,.bk-pipeline .container-title.REVIEWING .fold-atom-icon,.bk-pipeline .container-title.PREPARE_ENV .fold-atom-icon,.bk-pipeline .container-title.LOOP_WAITING .fold-atom-icon,.bk-pipeline .container-title.DEPENDENT_WAITING .fold-atom-icon,.bk-pipeline .container-title.CALL_WAITING .fold-atom-icon{color:#459fff}.bk-pipeline .container-title.CANCELED,.bk-pipeline .container-title.REVIEW_ABORT,.bk-pipeline .container-title.TRY_FINALLY,.bk-pipeline .container-title.QUEUE_CACHE{background-color:#f6b026}.bk-pipeline .container-title.CANCELED span.skip-name,.bk-pipeline .container-title.REVIEW_ABORT span.skip-name,.bk-pipeline .container-title.TRY_FINALLY span.skip-name,.bk-pipeline .container-title.QUEUE_CACHE span.skip-name{color:#fff}.bk-pipeline .container-title.CANCELED .fold-atom-icon,.bk-pipeline .container-title.REVIEW_ABORT .fold-atom-icon,.bk-pipeline .container-title.TRY_FINALLY .fold-atom-icon,.bk-pipeline .container-title.QUEUE_CACHE .fold-atom-icon{color:#f6b026}.bk-pipeline .container-title.FAILED,.bk-pipeline .container-title.TERMINATE,.bk-pipeline .container-title.HEARTBEAT_TIMEOUT,.bk-pipeline .container-title.QUALITY_CHECK_FAIL,.bk-pipeline .container-title.QUEUE_TIMEOUT,.bk-pipeline .container-title.EXEC_TIMEOUT{color:#fff;background-color:#ff5656}.bk-pipeline .container-title.FAILED .fold-atom-icon,.bk-pipeline .container-title.TERMINATE .fold-atom-icon,.bk-pipeline .container-title.HEARTBEAT_TIMEOUT .fold-atom-icon,.bk-pipeline .container-title.QUALITY_CHECK_FAIL .fold-atom-icon,.bk-pipeline .container-title.QUEUE_TIMEOUT .fold-atom-icon,.bk-pipeline .container-title.EXEC_TIMEOUT .fold-atom-icon{color:#ff5656}.bk-pipeline .container-title.SUCCEED,.bk-pipeline .container-title.REVIEW_PROCESSED,.bk-pipeline .container-title.STAGE_SUCCESS{color:#fff;background-color:#34d97b}.bk-pipeline .container-title.SUCCEED .fold-atom-icon,.bk-pipeline .container-title.REVIEW_PROCESSED .fold-atom-icon,.bk-pipeline .container-title.STAGE_SUCCESS .fold-atom-icon{color:#34d97b}.bk-pipeline .container-title.PAUSE{color:#fff;background-color:#ff9801}.bk-pipeline .container-title.PAUSE .fold-atom-icon{color:#ff9801}.bk-pipeline .bk-pipeline-atom.UNEXEC,.bk-pipeline .bk-pipeline-atom.SKIP,.bk-pipeline .bk-pipeline-atom.DISABLED{color:#63656e}.bk-pipeline .bk-pipeline-atom.CANCELED,.bk-pipeline .bk-pipeline-atom.REVIEW_ABORT,.bk-pipeline .bk-pipeline-atom.REVIEWING{border-color:#ffb400}.bk-pipeline .bk-pipeline-atom.CANCELED:before,.bk-pipeline .bk-pipeline-atom.REVIEW_ABORT:before,.bk-pipeline .bk-pipeline-atom.REVIEWING:before{background-color:#ffb400}.bk-pipeline .bk-pipeline-atom.CANCELED:after,.bk-pipeline .bk-pipeline-atom.REVIEW_ABORT:after,.bk-pipeline .bk-pipeline-atom.REVIEWING:after{border-color:#ffb400}.bk-pipeline .bk-pipeline-atom.CANCELED .atom-icon,.bk-pipeline .bk-pipeline-atom.CANCELED .atom-execute-time,.bk-pipeline .bk-pipeline-atom.CANCELED .stage-check-icon,.bk-pipeline .bk-pipeline-atom.REVIEW_ABORT .atom-icon,.bk-pipeline .bk-pipeline-atom.REVIEW_ABORT .atom-execute-time,.bk-pipeline .bk-pipeline-atom.REVIEW_ABORT .stage-check-icon,.bk-pipeline .bk-pipeline-atom.REVIEWING .atom-icon,.bk-pipeline .bk-pipeline-atom.REVIEWING .atom-execute-time,.bk-pipeline .bk-pipeline-atom.REVIEWING .stage-check-icon{color:#ffb400}.bk-pipeline .bk-pipeline-atom.FAILED,.bk-pipeline .bk-pipeline-atom.TERMINATE,.bk-pipeline .bk-pipeline-atom.HEARTBEAT_TIMEOUT,.bk-pipeline .bk-pipeline-atom.QUALITY_CHECK_FAIL,.bk-pipeline .bk-pipeline-atom.QUEUE_TIMEOUT,.bk-pipeline .bk-pipeline-atom.EXEC_TIMEOUT{border-color:#ff5656}.bk-pipeline .bk-pipeline-atom.FAILED:before,.bk-pipeline .bk-pipeline-atom.TERMINATE:before,.bk-pipeline .bk-pipeline-atom.HEARTBEAT_TIMEOUT:before,.bk-pipeline .bk-pipeline-atom.QUALITY_CHECK_FAIL:before,.bk-pipeline .bk-pipeline-atom.QUEUE_TIMEOUT:before,.bk-pipeline .bk-pipeline-atom.EXEC_TIMEOUT:before{background-color:#ff5656}.bk-pipeline .bk-pipeline-atom.FAILED:after,.bk-pipeline .bk-pipeline-atom.TERMINATE:after,.bk-pipeline .bk-pipeline-atom.HEARTBEAT_TIMEOUT:after,.bk-pipeline .bk-pipeline-atom.QUALITY_CHECK_FAIL:after,.bk-pipeline .bk-pipeline-atom.QUEUE_TIMEOUT:after,.bk-pipeline .bk-pipeline-atom.EXEC_TIMEOUT:after{border-color:#ff5656}.bk-pipeline .bk-pipeline-atom.FAILED .atom-icon,.bk-pipeline .bk-pipeline-atom.FAILED .atom-execute-time,.bk-pipeline .bk-pipeline-atom.FAILED .stage-check-icon,.bk-pipeline .bk-pipeline-atom.TERMINATE .atom-icon,.bk-pipeline .bk-pipeline-atom.TERMINATE .atom-execute-time,.bk-pipeline .bk-pipeline-atom.TERMINATE .stage-check-icon,.bk-pipeline .bk-pipeline-atom.HEARTBEAT_TIMEOUT .atom-icon,.bk-pipeline .bk-pipeline-atom.HEARTBEAT_TIMEOUT .atom-execute-time,.bk-pipeline .bk-pipeline-atom.HEARTBEAT_TIMEOUT .stage-check-icon,.bk-pipeline .bk-pipeline-atom.QUALITY_CHECK_FAIL .atom-icon,.bk-pipeline .bk-pipeline-atom.QUALITY_CHECK_FAIL .atom-execute-time,.bk-pipeline .bk-pipeline-atom.QUALITY_CHECK_FAIL .stage-check-icon,.bk-pipeline .bk-pipeline-atom.QUEUE_TIMEOUT .atom-icon,.bk-pipeline .bk-pipeline-atom.QUEUE_TIMEOUT .atom-execute-time,.bk-pipeline .bk-pipeline-atom.QUEUE_TIMEOUT .stage-check-icon,.bk-pipeline .bk-pipeline-atom.EXEC_TIMEOUT .atom-icon,.bk-pipeline .bk-pipeline-atom.EXEC_TIMEOUT .atom-execute-time,.bk-pipeline .bk-pipeline-atom.EXEC_TIMEOUT .stage-check-icon{color:#ff5656}.bk-pipeline .bk-pipeline-atom.SUCCEED,.bk-pipeline .bk-pipeline-atom.REVIEW_PROCESSED{border-color:#34d97b}.bk-pipeline .bk-pipeline-atom.SUCCEED:before,.bk-pipeline .bk-pipeline-atom.REVIEW_PROCESSED:before{background-color:#34d97b}.bk-pipeline .bk-pipeline-atom.SUCCEED:after,.bk-pipeline .bk-pipeline-atom.REVIEW_PROCESSED:after{border-color:#34d97b}.bk-pipeline .bk-pipeline-atom.SUCCEED .atom-icon,.bk-pipeline .bk-pipeline-atom.SUCCEED .atom-execute-time,.bk-pipeline .bk-pipeline-atom.SUCCEED .stage-check-icon,.bk-pipeline .bk-pipeline-atom.REVIEW_PROCESSED .atom-icon,.bk-pipeline .bk-pipeline-atom.REVIEW_PROCESSED .atom-execute-time,.bk-pipeline .bk-pipeline-atom.REVIEW_PROCESSED .stage-check-icon{color:#34d97b}.bk-pipeline .bk-pipeline-atom.SUCCEED .stage-check-icon,.bk-pipeline .bk-pipeline-atom.REVIEW_PROCESSED .stage-check-icon{border-color:#34d97b}.bk-pipeline .bk-pipeline-atom.PAUSE{border-color:#ff9801;color:#ff9801}.bk-pipeline .bk-pipeline-atom.PAUSE:before{background-color:#ff9801}.bk-pipeline .bk-pipeline-atom.PAUSE:after{border-color:#ff9801}.bk-pipeline .bk-pipeline-atom.PAUSE .atom-icon,.bk-pipeline .bk-pipeline-atom.PAUSE .atom-execute-time{color:#63656e}.bk-pipeline .bk-pipeline-atom.template-compare-atom{border-color:#ff6e00}.bk-pipeline .bk-pipeline-atom.template-compare-atom:before{background-color:#ff6e00}.bk-pipeline .bk-pipeline-atom.template-compare-atom:after{border-color:#ff6e00}.bk-pipeline .bk-pipeline-atom.template-compare-atom:hover{border-color:#ffb400}.bk-pipeline .bk-pipeline-atom.quality-atom{background-color:rgba(0,0,0,0)}.bk-pipeline .bk-pipeline-atom.quality-atom.SUCCEED{border-color:#34d97b}.bk-pipeline .bk-pipeline-atom.quality-atom.SUCCEED .atom-title{color:#34d97b}.bk-pipeline .bk-pipeline-atom.quality-atom.SUCCEED .atom-title>i{border-color:#34d97b}.bk-pipeline .bk-pipeline-atom.quality-atom.REVIEWING{border-color:#ffb400}.bk-pipeline .bk-pipeline-atom.quality-atom.REVIEWING .atom-title{color:#ffb400}.bk-pipeline .bk-pipeline-atom.quality-atom.REVIEWING .atom-title>i{border-color:#ffb400}.bk-pipeline .bk-pipeline-atom.quality-atom.REVIEWING .atom-title>i:last-child{border-color:rgba(0,0,0,0)}.bk-pipeline .bk-pipeline-atom.quality-atom.FAILED{border-color:#ff5656}.bk-pipeline .bk-pipeline-atom.quality-atom.FAILED .atom-title{color:#ff5656}.bk-pipeline .bk-pipeline-atom.quality-atom.FAILED .atom-title>i{border-top:2px solid #ff5656}',""]);const s=r},29:(e,t,i)=>{"use strict";i.d(t,{A:()=>s});var n=i(441),o=i.n(n),a=i(562),r=i.n(a)()(o());r.push([e.id,'/*!\n * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.\n *\n * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.\n *\n * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.\n *\n * A copy of the MIT License is included in this file.\n *\n *\n * Terms of the MIT License:\n * ---------------------------------------------------\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */@keyframes rotating{from{transform:rotate(0)}to{transform:rotate(359deg)}}.bk-pipeline .bk-pipeline-atom{cursor:pointer;position:relative;display:flex;flex-direction:row;align-items:center;height:42px;margin:0 0 11px 0;background-color:#fff;border-radius:2px;font-size:14px;transition:all .4s ease-in-out;z-index:2;border:1px solid #c4cdd6}.bk-pipeline .bk-pipeline-atom .atom-progress{display:inline-flex;width:42px;height:42px;align-items:center;justify-content:center}.bk-pipeline .bk-pipeline-atom .active-atom-location-icon{position:absolute;color:#3c96ff;left:-30px}.bk-pipeline .bk-pipeline-atom.trigger-atom:before,.bk-pipeline .bk-pipeline-atom.trigger-atom:after{display:none}.bk-pipeline .bk-pipeline-atom:first-child:before{top:-16px}.bk-pipeline .bk-pipeline-atom:before{content:"";position:absolute;height:14px;width:2px;background:#c4cdd6;top:-12px;left:22px;z-index:1}.bk-pipeline .bk-pipeline-atom:after{content:"";position:absolute;height:4px;width:4px;border:2px solid #c4cdd6;border-radius:50%;background:#fff !important;top:-5px;left:19px;z-index:2}.bk-pipeline .bk-pipeline-atom.is-intercept{border-color:#ffb400}.bk-pipeline .bk-pipeline-atom.is-intercept:hover{border-color:#ffb400}.bk-pipeline .bk-pipeline-atom.is-error{border-color:#ff5656;color:#ff5656}.bk-pipeline .bk-pipeline-atom.is-error:hover .atom-invalid-icon{display:none}.bk-pipeline .bk-pipeline-atom.is-error .atom-invalid-icon{margin:0 12px}.bk-pipeline .bk-pipeline-atom:not(.readonly):hover{border-color:#3c96ff}.bk-pipeline .bk-pipeline-atom:not(.readonly):hover .atom-icon.skip-icon{color:#c4cdd6}.bk-pipeline .bk-pipeline-atom:not(.readonly):hover .atom-icon,.bk-pipeline .bk-pipeline-atom:not(.readonly):hover .atom-name{color:#3c96ff}.bk-pipeline .bk-pipeline-atom:not(.readonly):hover .add-plus-icon.close,.bk-pipeline .bk-pipeline-atom:not(.readonly):hover .copy{cursor:pointer;display:block}.bk-pipeline .bk-pipeline-atom .atom-icon{text-align:center;margin:0 14.5px;font-size:18px;width:18px;color:#63656e;fill:currentColor}.bk-pipeline .bk-pipeline-atom .atom-icon.skip-icon{color:#c4cdd6}.bk-pipeline .bk-pipeline-atom .atom-name span.skip-name{text-decoration:line-through;color:#c4cdd6}.bk-pipeline .bk-pipeline-atom .atom-name span.skip-name:hover{color:#c4cdd6}.bk-pipeline .bk-pipeline-atom .pause-button{margin-right:8px;color:#3c96ff}.bk-pipeline .bk-pipeline-atom .add-plus-icon.close{position:relative;display:block;width:16px;height:16px;border:1px solid #fff;background-color:#c4c6cd;border-radius:50%;transition:all .3s ease;display:none;margin-right:10px;border:none;transform:rotate(45deg)}.bk-pipeline .bk-pipeline-atom .add-plus-icon.close:before,.bk-pipeline .bk-pipeline-atom .add-plus-icon.close:after{content:"";position:absolute;left:7px;top:4px;left:6px;top:3px;height:8px;width:2px;background-color:#fff}.bk-pipeline .bk-pipeline-atom .add-plus-icon.close:after{transform:rotate(90deg)}.bk-pipeline .bk-pipeline-atom .add-plus-icon.close:hover{border-color:#ff5656;background-color:#ff5656}.bk-pipeline .bk-pipeline-atom .add-plus-icon.close:hover:before,.bk-pipeline .bk-pipeline-atom .add-plus-icon.close:hover:after{background-color:#fff}.bk-pipeline .bk-pipeline-atom .add-plus-icon.close:before,.bk-pipeline .bk-pipeline-atom .add-plus-icon.close:after{left:7px;top:4px}.bk-pipeline .bk-pipeline-atom .copy{display:none;margin-right:10px;color:#c4cdd6}.bk-pipeline .bk-pipeline-atom .copy:hover{color:#3c96ff}.bk-pipeline .bk-pipeline-atom>.atom-name{flex:1;color:#63656e;display:inline-block;max-width:auto;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;max-width:188px;margin-right:2px}.bk-pipeline .bk-pipeline-atom>.atom-name span:hover{color:#3c96ff}.bk-pipeline .bk-pipeline-atom .disabled{cursor:not-allowed;color:#c4cdd6}.bk-pipeline .bk-pipeline-atom .atom-execounter{color:#3c96ff;font-size:12px}.bk-pipeline .bk-pipeline-atom .atom-operate-area{margin:0 8px 0 0;color:#3c96ff;font-size:12px}.bk-pipeline .bk-pipeline-atom .atom-reviewing-tips[disabled]{cursor:not-allowed;color:#c3cdd7}.bk-pipeline .bk-pipeline-atom .atom-review-diasbled-tips{color:#c3cdd7;margin:0 8px 0 2px}.bk-pipeline .bk-pipeline-atom .atom-canskip-checkbox{margin-right:6px}.bk-pipeline .bk-pipeline-atom.quality-atom{display:flex;justify-content:center;border-color:rgba(0,0,0,0);height:24px;background:rgba(0,0,0,0);border-color:rgba(0,0,0,0) !important;font-size:12px}.bk-pipeline .bk-pipeline-atom.quality-atom:before{height:40px;z-index:8}.bk-pipeline .bk-pipeline-atom.quality-atom:after{display:none}.bk-pipeline .bk-pipeline-atom.quality-atom.last-quality-atom:before{height:22px}.bk-pipeline .bk-pipeline-atom.quality-atom .atom-title{display:flex;width:100%;align-items:center;justify-content:center;margin-left:22px}.bk-pipeline .bk-pipeline-atom.quality-atom .atom-title>span{border-radius:12px;font-weight:bold;border:1px solid #c4cdd6;padding:0 12px;margin:0 4px}.bk-pipeline .bk-pipeline-atom.quality-atom .atom-title>i{height:0;flex:1;border-top:2px dashed #c4cdd6}.bk-pipeline .bk-pipeline-atom.quality-atom .handler-list{position:absolute;right:0}.bk-pipeline .bk-pipeline-atom.quality-atom .handler-list span{color:#3c96ff;font-size:12px}.bk-pipeline .bk-pipeline-atom.quality-atom .handler-list span:first-child{margin-right:5px}.bk-pipeline .bk-pipeline-atom.quality-atom .executing-job{position:absolute;top:6px;right:42px}.bk-pipeline .bk-pipeline-atom.quality-atom .executing-job:before{display:inline-block;animation:rotating infinite .6s ease-in-out}.bk-pipeline .bk-pipeline-atom.quality-atom .disabled-review span{color:#c4cdd6;cursor:default}.bk-pipeline .bk-pipeline-atom.readonly{background-color:#fff}.bk-pipeline .bk-pipeline-atom.readonly .atom-name:hover span{color:#63656e}.bk-pipeline .bk-pipeline-atom.readonly .atom-name:hover .skip-name{text-decoration:line-through;color:#c4cdd6}.bk-pipeline .bk-pipeline-atom.readonly.quality-prev-atom:before{height:24px;top:-23px}',""]);const s=r},259:(e,t,i)=>{"use strict";i.d(t,{A:()=>s});var n=i(441),o=i.n(n),a=i(562),r=i.n(a)()(o());r.push([e.id,'/*!\n * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.\n *\n * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.\n *\n * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.\n *\n * A copy of the MIT License is included in this file.\n *\n *\n * Terms of the MIT License:\n * ---------------------------------------------------\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */@keyframes rotating{from{transform:rotate(0)}to{transform:rotate(359deg)}}.container-atom-list{position:relative;z-index:3}.container-atom-list .sortable-ghost-atom{opacity:.5}.container-atom-list .sortable-chosen-atom{transform:scale(1)}.container-atom-list .add-atom-entry{position:absolute;bottom:-10px;left:calc(50% - 9px);cursor:pointer;z-index:3}.container-atom-list .add-atom-entry .add-plus-icon{position:relative;display:block;width:18px;height:18px;border:1px solid #c4cdd6;background-color:#fff;border-radius:50%;transition:all .3s ease}.container-atom-list .add-atom-entry .add-plus-icon:before,.container-atom-list .add-atom-entry .add-plus-icon:after{content:"";position:absolute;left:8px;top:5px;left:7px;top:4px;height:8px;width:2px;background-color:#c4cdd6}.container-atom-list .add-atom-entry .add-plus-icon:after{transform:rotate(90deg)}.container-atom-list .add-atom-entry .add-plus-icon:hover{border-color:#3c96ff;background-color:#3c96ff}.container-atom-list .add-atom-entry .add-plus-icon:hover:before,.container-atom-list .add-atom-entry .add-plus-icon:hover:after{background-color:#fff}.container-atom-list .add-atom-entry.block-add-entry{display:flex;flex-direction:row;align-items:center;height:42px;margin:0 0 11px 0;background-color:#fff;border-radius:2px;font-size:14px;transition:all .4s ease-in-out;z-index:2;position:static;padding-right:12px;border-style:dashed;color:#ff5656;border-color:#ff5656;border-width:1px}.container-atom-list .add-atom-entry.block-add-entry .add-atom-label{flex:1;color:#dde4eb}.container-atom-list .add-atom-entry.block-add-entry .add-plus-icon{margin:12px 13px}.container-atom-list .add-atom-entry.block-add-entry:before,.container-atom-list .add-atom-entry.block-add-entry:after{display:none}.container-atom-list .add-atom-entry:hover{border-color:#3c96ff;color:#3c96ff}.container-atom-list .post-action-arrow{position:relativecd;display:flex;align-items:center;justify-content:center;position:absolute;height:14px;width:14px;border:1px solid #c4cdd6;color:#c4cdd6;border-radius:50%;background:#fff !important;top:-7px;left:17px;z-index:3;font-weight:bold}.container-atom-list .post-action-arrow .toggle-post-action-icon{display:block;transition:all .5s ease}.container-atom-list .post-action-arrow.post-action-arrow-show .toggle-post-action-icon{transform:rotate(180deg)}.container-atom-list .post-action-arrow::after{content:"";position:absolute;width:2px;height:6px;background-color:#c4cdd6;left:5px;top:-6px}.container-atom-list .post-action-arrow.FAILED{border-color:#ff5656;color:#ff5656}.container-atom-list .post-action-arrow.FAILED::after{background-color:#ff5656}.container-atom-list .post-action-arrow.CANCELED{border-color:#ffb400;color:#ffb400}.container-atom-list .post-action-arrow.CANCELED::after{background-color:#ffb400}.container-atom-list .post-action-arrow.SUCCEED{border-color:#34d97b;color:#34d97b}.container-atom-list .post-action-arrow.SUCCEED::after{background-color:#34d97b}.container-atom-list .post-action-arrow.RUNNING{border-color:#3c96ff;color:#3c96ff}.container-atom-list .post-action-arrow.RUNNING::after{background-color:#3c96ff}',""]);const s=r},162:(e,t,i)=>{"use strict";i.d(t,{A:()=>s});var n=i(441),o=i.n(n),a=i(562),r=i.n(a)()(o());r.push([e.id,".is-danger[data-v-8b5d140e]{color:#ff5656}",""]);const s=r},662:(e,t,i)=>{"use strict";i.d(t,{A:()=>s});var n=i(441),o=i.n(n),a=i(562),r=i.n(a)()(o());r.push([e.id,'/*!\n * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.\n *\n * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.\n *\n * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.\n *\n * A copy of the MIT License is included in this file.\n *\n *\n * Terms of the MIT License:\n * ---------------------------------------------------\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */@keyframes rotating{from{transform:rotate(0)}to{transform:rotate(359deg)}}.devops-stage-container .container-title{display:flex;height:42px;background:#33333f;cursor:pointer;color:#fff;font-size:14px;align-items:center;position:relative;margin:0 0 16px 0;z-index:3}.devops-stage-container .container-title>.container-name{display:inline-block;max-width:auto;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;flex:1;padding:0 6px}.devops-stage-container .container-title .atom-canskip-checkbox{margin-right:6px}.devops-stage-container .container-title .atom-canskip-checkbox.is-disabled .bk-checkbox{background-color:rgba(0,0,0,0);border-color:#979ba4}.devops-stage-container .container-title input[type=checkbox]{border-radius:3px}.devops-stage-container .container-title .matrix-flag-icon{position:absolute;top:0px;font-size:16px}.devops-stage-container .container-title .fold-atom-icon{position:absolute;background:#fff;border-radius:50%;bottom:-10px;left:calc(50% - 9px);color:#c4cdd6;transition:all .3s ease}.devops-stage-container .container-title .fold-atom-icon.open{transform:rotate(-180deg)}.devops-stage-container .container-title .copyJob{display:none;margin-right:10px;color:#c4cdd6;cursor:pointer}.devops-stage-container .container-title .copyJob:hover{color:#3c96ff}.devops-stage-container .container-title .close{position:relative;display:block;width:16px;height:16px;border:1px solid #2e2e3a;background-color:#c4c6cd;border-radius:50%;transition:all .3s ease;border:none;display:none;margin-right:10px;transform:rotate(45deg);cursor:pointer}.devops-stage-container .container-title .close:before,.devops-stage-container .container-title .close:after{content:"";position:absolute;left:7px;top:4px;left:6px;top:3px;height:8px;width:2px;background-color:#2e2e3a}.devops-stage-container .container-title .close:after{transform:rotate(90deg)}.devops-stage-container .container-title .close:hover{border-color:#ff5656;background-color:#ff5656}.devops-stage-container .container-title .close:hover:before,.devops-stage-container .container-title .close:hover:after{background-color:#fff}.devops-stage-container .container-title .close:before,.devops-stage-container .container-title .close:after{left:7px;top:4px}.devops-stage-container .container-title .debug-btn{position:absolute;height:100%;right:0}.devops-stage-container .container-title .container-locate-icon{position:absolute;left:-30px;top:13px;color:#3c96ff}.devops-stage-container .container-title:hover .copyJob,.devops-stage-container .container-title:hover .close{display:block}.devops-stage-container .container-title:hover .hover-hide{display:none}',""]);const s=r},637:(e,t,i)=>{"use strict";i.d(t,{A:()=>s});var n=i(441),o=i.n(n),a=i(562),r=i.n(a)()(o());r.push([e.id,'/*!\n * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.\n *\n * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.\n *\n * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.\n *\n * A copy of the MIT License is included in this file.\n *\n *\n * Terms of the MIT License:\n * ---------------------------------------------------\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */@keyframes rotating{from{transform:rotate(0)}to{transform:rotate(359deg)}}.bk-pipeline-matrix-group{border:1px solid #b5c0d5;padding:10px;background:#fff}.bk-pipeline-matrix-group .bk-pipeline-matrix-group-header{display:flex;align-items:center;cursor:pointer;justify-content:space-between;height:20px}.bk-pipeline-matrix-group .bk-pipeline-matrix-group-header .matrix-name{display:flex;align-items:center;font-size:14px;color:#222;min-width:0}.bk-pipeline-matrix-group .bk-pipeline-matrix-group-header .matrix-name .matrix-fold-icon{display:block;margin-right:10px;transition:all .3s ease;flex-shrink:0}.bk-pipeline-matrix-group .bk-pipeline-matrix-group-header .matrix-name .matrix-fold-icon.open{transform:rotate(-180deg)}.bk-pipeline-matrix-group .bk-pipeline-matrix-group-header .matrix-name>span{display:inline-block;max-width:auto;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.bk-pipeline-matrix-group .bk-pipeline-matrix-group-header .matrix-status{color:#3c96ff;display:flex;align-items:center}.bk-pipeline-matrix-group .bk-pipeline-matrix-group-header .matrix-status .status-desc{font-size:12px;display:inline-block;max-width:110px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.bk-pipeline-matrix-group .matrix-body{margin-top:12px}.bk-pipeline-matrix-group .matrix-body>div{margin-bottom:34px}',""]);const s=r},533:(e,t,i)=>{"use strict";i.d(t,{A:()=>s});var n=i(441),o=i.n(n),a=i(562),r=i.n(a)()(o());r.push([e.id,'/*!\n * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.\n *\n * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.\n *\n * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.\n *\n * A copy of the MIT License is included in this file.\n *\n *\n * Terms of the MIT License:\n * ---------------------------------------------------\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */@keyframes rotating{from{transform:rotate(0)}to{transform:rotate(359deg)}}.pipeline-drag{cursor:grab,default}.pipeline-stage{position:relative;width:280px;border-radius:2px;padding:0;background:#f5f5f5;margin:0 80px 0 0}.pipeline-stage .pipeline-stage-entry{position:relative;cursor:pointer;display:flex;width:100%;height:50px;align-items:center;min-width:0;font-size:14px;background-color:#eff5ff;border:1px solid #d4e8ff;color:#3c96ff}.pipeline-stage .pipeline-stage-entry:hover{border-color:#1a6df3;background-color:#d1e2fd}.pipeline-stage .pipeline-stage-entry .check-in-icon,.pipeline-stage .pipeline-stage-entry .check-out-icon{position:absolute;left:-14px;top:11px}.pipeline-stage .pipeline-stage-entry .check-in-icon.check-out-icon,.pipeline-stage .pipeline-stage-entry .check-out-icon.check-out-icon{left:auto;right:-14px}.pipeline-stage .pipeline-stage-entry .stage-entry-name{flex:1;display:flex;align-items:center;justify-content:center;margin:0 80px;overflow:hidden}.pipeline-stage .pipeline-stage-entry .stage-entry-name .stage-title-name{display:inline-block;max-width:auto;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;margin-left:6px}.pipeline-stage .pipeline-stage-entry .stage-single-retry{cursor:pointer;position:absolute;right:6%;color:#3c96ff}.pipeline-stage .pipeline-stage-entry .stage-entry-error-icon,.pipeline-stage .pipeline-stage-entry .check-total-stage{position:absolute;right:27px}.pipeline-stage .pipeline-stage-entry .stage-entry-error-icon.stage-entry-error-icon,.pipeline-stage .pipeline-stage-entry .check-total-stage.stage-entry-error-icon{top:16px;right:8px;color:#ff5656}.pipeline-stage .pipeline-stage-entry .stage-entry-btns{position:absolute;right:0;top:16px;display:none;width:80px;align-items:center;justify-content:flex-end;color:#fff;fill:#fff;z-index:2}.pipeline-stage .pipeline-stage-entry .stage-entry-btns .copy-stage:hover{color:#3c96ff}.pipeline-stage .pipeline-stage-entry .stage-entry-btns .close{position:relative;display:block;width:16px;height:16px;border:1px solid #2e2e3a;background-color:#fff;border-radius:50%;transition:all .3s ease;border:none;margin:0 10px 0 8px;transform:rotate(45deg);cursor:pointer}.pipeline-stage .pipeline-stage-entry .stage-entry-btns .close:before,.pipeline-stage .pipeline-stage-entry .stage-entry-btns .close:after{content:"";position:absolute;left:7px;top:4px;left:6px;top:3px;height:8px;width:2px;background-color:#2e2e3a}.pipeline-stage .pipeline-stage-entry .stage-entry-btns .close:after{transform:rotate(90deg)}.pipeline-stage .pipeline-stage-entry .stage-entry-btns .close:hover{border-color:#ff5656;background-color:#ff5656}.pipeline-stage .pipeline-stage-entry .stage-entry-btns .close:hover:before,.pipeline-stage .pipeline-stage-entry .stage-entry-btns .close:hover:after{background-color:#fff}.pipeline-stage .pipeline-stage-entry .stage-entry-btns .close:before,.pipeline-stage .pipeline-stage-entry .stage-entry-btns .close:after{left:7px;top:4px}.pipeline-stage.editable:not(.readonly) .pipeline-stage-entry:hover{color:#000;border-color:#1a6df3;background-color:#d1e2fd}.pipeline-stage.editable .pipeline-stage-entry:hover .stage-entry-btns{display:flex}.pipeline-stage.editable .pipeline-stage-entry:hover .stage-entry-error-icon{display:none}.pipeline-stage.readonly.SKIP .pipeline-stage-entry{color:#c3cdd7;fill:#c3cdd7}.pipeline-stage.readonly.RUNNING .pipeline-stage-entry{background-color:#eff5ff;border-color:#d4e8ff;color:#3c96ff}.pipeline-stage.readonly.REVIEWING .pipeline-stage-entry{background-color:#f3f3f3;border-color:#d0d8ea;color:#000}.pipeline-stage.readonly.FAILED .pipeline-stage-entry{border-color:#ffd4d4;background-color:#fff9f9;color:#000}.pipeline-stage.readonly.SUCCEED .pipeline-stage-entry{background-color:#f3fff6;border-color:#bbefc9;color:#000}.pipeline-stage.readonly .pipeline-stage-entry{background-color:#f3f3f3;border-color:#d0d8ea;color:#000}.pipeline-stage.readonly .pipeline-stage-entry .skip-icon{vertical-align:middle}.pipeline-stage .add-connector{stroke-dasharray:4,4;top:7px;left:10px}.pipeline-stage .append-stage{position:absolute;top:16px;right:-44px;z-index:3}.pipeline-stage .append-stage .add-plus-icon{box-shadow:0px 2px 4px 0px rgba(60,150,255,.2)}.pipeline-stage .append-stage .line-add{top:-46px;left:-16px}.pipeline-stage .append-stage .add-plus-connector{position:absolute;width:40px;height:2px;left:-26px;top:8px;background-color:#3c96ff}.pipeline-stage .add-menu{position:absolute;top:16px;left:-53px;cursor:pointer;z-index:3}.pipeline-stage .add-menu .add-plus-icon{box-shadow:0px 2px 4px 0px rgba(60,150,255,.2)}.pipeline-stage .add-menu .minus-icon{z-index:4}.pipeline-stage .add-menu .line-add{top:-46px;left:-16px}.pipeline-stage .add-menu .parallel-add{left:50px}.pipeline-stage .add-menu .add-plus-connector{position:absolute;width:24px;height:2px;left:17px;top:8px;background-color:#3c96ff}.pipeline-stage:first-child .stage-connector{display:none}.pipeline-stage.is-final-stage .stage-connector{width:80px}.pipeline-stage .stage-connector{position:absolute;width:66px;height:2px;left:-80px;top:24px;color:#3c96ff;background-color:#3c96ff}.pipeline-stage .stage-connector:before{content:"";width:8px;height:8px;position:absolute;left:-4px;top:-3px;background-color:#3c96ff;border-radius:50%}.pipeline-stage .stage-connector .connector-angle{position:absolute;right:-3px;top:-6px}.pipeline-stage .insert-stage{position:absolute;display:block;width:160px;background-color:#fff;border:1px solid #dcdee5}.pipeline-stage .insert-stage .click-item{padding:0 15px;font-size:12px;line-height:32px}.pipeline-stage .insert-stage .click-item:hover,.pipeline-stage .insert-stage .click-item :hover{color:#3c96ff;background-color:#eaf3ff}.pipeline-stage .insert-stage .disabled-item{cursor:not-allowed;color:#c4cdd6}.pipeline-stage .insert-stage .disabled-item:hover,.pipeline-stage .insert-stage .disabled-item :hover{color:#c4cdd6;background-color:#fff}.stage-retry-dialog .bk-form-radio{display:block;margin-top:15px}.stage-retry-dialog .bk-form-radio .bk-radio-text{font-size:14px}',""]);const s=r},935:(e,t,i)=>{"use strict";i.d(t,{A:()=>s});var n=i(441),o=i.n(n),a=i(562),r=i.n(a)()(o());r.push([e.id,'/*!\n * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.\n *\n * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.\n *\n * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.\n *\n * A copy of the MIT License is included in this file.\n *\n *\n * Terms of the MIT License:\n * ---------------------------------------------------\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */@keyframes rotating{from{transform:rotate(0)}to{transform:rotate(359deg)}}.stage-check-icon{border-radius:100px;border:1px solid #d0d8ea;display:flex;align-items:center;background:#fff;font-size:12px;z-index:3;color:#63656e}.stage-check-icon.is-readonly-check-icon{filter:grayscale(100%)}.stage-check-icon.reviewing{color:#3c96ff;border-color:#3c96ff}.stage-check-icon.quality-check{color:#34d97b;border-color:#34d97b}.stage-check-icon.quality-check-error,.stage-check-icon.review-error{color:#ff5656;border-color:#ff5656}.stage-check-icon .stage-check-txt{padding-right:10px}',""]);const s=r},49:(e,t,i)=>{"use strict";i.d(t,{A:()=>s});var n=i(441),o=i.n(n),a=i(562),r=i.n(a)()(o());r.push([e.id,'/*!\n * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.\n *\n * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.\n *\n * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.\n *\n * A copy of the MIT License is included in this file.\n *\n *\n * Terms of the MIT License:\n * ---------------------------------------------------\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */@keyframes rotating{from{transform:rotate(0)}to{transform:rotate(359deg)}}.devops-stage-container{text-align:left;margin:16px 20px 24px 20px;position:relative}.devops-stage-container:not(.last-stage-container):after{content:"";width:6px;height:6px;position:absolute;right:-3px;top:19px;border-radius:50%}.devops-stage-container:not(.last-stage-container):after:not(.readonly){background:#3c96ff}.devops-stage-container .container-connect-triangle{position:absolute;color:#3c96ff;left:-9px;top:15.5px;z-index:2}.devops-stage-container .connect-line{position:absolute;top:1px;stroke:#3c96ff;stroke-width:1;fill:none;z-index:0}.devops-stage-container .connect-line.left{left:-54px}.devops-stage-container .connect-line.right{right:-46px}.devops-stage-container .connect-line.first-connect-line{height:76px;width:58px;top:-43px}.devops-stage-container .connect-line.first-connect-line.left{left:-63px}.devops-stage-container .connect-line.first-connect-line.right{left:auto;right:-55px}',""]);const s=r},82:(e,t,i)=>{"use strict";i.d(t,{A:()=>s});var n=i(441),o=i.n(n),a=i(562),r=i.n(a)()(o());r.push([e.id,'/*!\n * Tencent is pleased to support the open source community by making BK-CI 蓝鲸持续集成平台 available.\n *\n * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.\n *\n * BK-CI 蓝鲸持续集成平台 is licensed under the MIT license.\n *\n * A copy of the MIT License is included in this file.\n *\n *\n * Terms of the MIT License:\n * ---------------------------------------------------\n * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n */@keyframes rotating{from{transform:rotate(0)}to{transform:rotate(359deg)}}.stage-status{position:relative;text-align:center;overflow:hidden;font-size:14px;width:42px;height:42px;box-sizing:border-box}.stage-status>span{position:absolute;width:100%;height:100%;display:flex;align-items:center;justify-content:center;left:0;top:0;transition:all .3s cubic-bezier(1, 0.5, 0.8, 1)}.stage-status.matrix{width:20px;height:20px}.stage-status .status-logo{position:absolute;left:15px;top:15px}.stage-status .slide-top-enter,.stage-status .slide-top-leave-to{transform:translateY(42px)}.stage-status .slide-down-enter,.stage-status .slide-down-leave-to{transform:translateY(-42px)}.stage-status .slide-left-enter,.stage-status .slide-left-leave-to{transform:translateX(42px)}.stage-status .slide-right-enter,.stage-status .slide-right-leave-to{transform:translateX(-42px)}.stage-status.readonly{font-size:12px;font-weight:normal;background-color:rgba(0,0,0,0)}.stage-status.readonly.container{color:#fff}',""]);const s=r},157:(e,t,i)=>{"use strict";i.d(t,{A:()=>s});var n=i(441),o=i.n(n),a=i(562),r=i.n(a)()(o());r.push([e.id,".bk-pipeline{display:flex;padding-right:120px;width:fit-content;position:relative;align-items:flex-start}.bk-pipeline ul,.bk-pipeline li{margin:0;padding:0}.list-item{transition:transform .2s ease-out}.list-enter,.list-leave-to{opacity:0;transform:translateY(36px) scale(0, 1)}.list-leave-active{position:absolute !important}",""]);const s=r},562:e=>{"use strict";e.exports=function(e){var t=[];return t.toString=function(){return this.map((function(t){var i="",n=void 0!==t[5];return t[4]&&(i+="@supports (".concat(t[4],") {")),t[2]&&(i+="@media ".concat(t[2]," {")),n&&(i+="@layer".concat(t[5].length>0?" ".concat(t[5]):""," {")),i+=e(t),n&&(i+="}"),t[2]&&(i+="}"),t[4]&&(i+="}"),i})).join("")},t.i=function(e,i,n,o,a){"string"==typeof e&&(e=[[null,e,void 0]]);var r={};if(n)for(var s=0;s0?" ".concat(p[5]):""," {").concat(p[1],"}")),p[5]=a),i&&(p[2]?(p[1]="@media ".concat(p[2]," {").concat(p[1],"}"),p[2]=i):p[2]=i),o&&(p[4]?(p[1]="@supports (".concat(p[4],") {").concat(p[1],"}"),p[4]=o):p[4]="".concat(o)),t.push(p))}},t}},441:e=>{"use strict";e.exports=function(e){return e[1]}},713:(e,t,i)=>{"use strict";function n(e){return n="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(e){return typeof e}:function(e){return e&&"function"==typeof Symbol&&e.constructor===Symbol&&e!==Symbol.prototype?"symbol":typeof e},n(e)}function o(e,t,i){return t in e?Object.defineProperty(e,t,{value:i,enumerable:!0,configurable:!0,writable:!0}):e[t]=i,e}function a(){return a=Object.assign||function(e){for(var t=1;tgt,Sortable:()=>ze,Swap:()=>at,default:()=>yt});var l=s(/(?:Trident.*rv[ :]?11\.|msie|iemobile|Windows Phone)/i),c=s(/Edge/i),p=s(/firefox/i),d=s(/safari/i)&&!s(/chrome/i)&&!s(/android/i),u=s(/iP(ad|od|hone)/i),h=s(/chrome/i)&&s(/android/i),f={capture:!1,passive:!1};function m(e,t,i){e.addEventListener(t,i,!l&&f)}function g(e,t,i){e.removeEventListener(t,i,!l&&f)}function b(e,t){if(t){if(">"===t[0]&&(t=t.substring(1)),e)try{if(e.matches)return e.matches(t);if(e.msMatchesSelector)return e.msMatchesSelector(t);if(e.webkitMatchesSelector)return e.webkitMatchesSelector(t)}catch(e){return!1}return!1}}function v(e){return e.host&&e!==document&&e.host.nodeType?e.host:e.parentNode}function y(e,t,i,n){if(e){i=i||document;do{if(null!=t&&(">"===t[0]?e.parentNode===i&&b(e,t):b(e,t))||n&&e===i)return e;if(e===i)break}while(e=v(e))}return null}var E,x=/\s+/g;function k(e,t,i){if(e&&t)if(e.classList)e.classList[i?"add":"remove"](t);else{var n=(" "+e.className+" ").replace(x," ").replace(" "+t+" "," ");e.className=(n+(i?" "+t:"")).replace(x," ")}}function I(e,t,i){var n=e&&e.style;if(n){if(void 0===i)return document.defaultView&&document.defaultView.getComputedStyle?i=document.defaultView.getComputedStyle(e,""):e.currentStyle&&(i=e.currentStyle),void 0===t?i:i[t];t in n||-1!==t.indexOf("webkit")||(t="-webkit-"+t),n[t]=i+("string"==typeof i?"":"px")}}function T(e,t){var i="";if("string"==typeof e)i=e;else do{var n=I(e,"transform");n&&"none"!==n&&(i=n+" "+i)}while(!t&&(e=e.parentNode));var o=window.DOMMatrix||window.WebKitCSSMatrix||window.CSSMatrix||window.MSCSSMatrix;return o&&new o(i)}function S(e,t,i){if(e){var n=e.getElementsByTagName(t),o=0,a=n.length;if(i)for(;o=a:o<=a))return n;if(n===C())break;n=M(n,!1)}return!1}function O(e,t,i){for(var n=0,o=0,a=e.children;o2&&void 0!==arguments[2]?arguments[2]:{},n=i.evt,o=function(e,t){if(null==e)return{};var i,n,o=function(e,t){if(null==e)return{};var i,n,o={},a=Object.keys(e);for(n=0;n=0||(o[i]=e[i]);return o}(e,t);if(Object.getOwnPropertySymbols){var a=Object.getOwnPropertySymbols(e);for(n=0;n=0||Object.prototype.propertyIsEnumerable.call(e,i)&&(o[i]=e[i])}return o}(i,["evt"]);W.pluginEvent.bind(ze)(e,t,r({dragEl:Y,parentEl:q,ghostEl:K,rootEl:X,nextEl:Q,lastDownEl:J,cloneEl:Z,cloneHidden:ee,dragStarted:he,putSortable:re,activeSortable:ze.active,originalEvent:n,oldIndex:te,oldDraggableIndex:ne,newIndex:ie,newDraggableIndex:oe,hideGhostForTarget:Me,unhideGhostForTarget:Le,cloneNowHidden:function(){ee=!0},cloneNowShown:function(){ee=!1},dispatchSortableEvent:function(e){$({sortable:t,name:e,originalEvent:n})}},o))};function $(e){j(r({putSortable:re,cloneEl:Z,targetEl:Y,rootEl:X,oldIndex:te,oldDraggableIndex:ne,newIndex:ie,newDraggableIndex:oe},e))}var Y,q,K,X,Q,J,Z,ee,te,ie,ne,oe,ae,re,se,le,ce,pe,de,ue,he,fe,me,ge,be,ve=!1,ye=!1,Ee=[],xe=!1,ke=!1,Ie=[],Te=!1,Se=[],Ce="undefined"!=typeof document,Ae=u,we=c||l?"cssFloat":"float",Oe=Ce&&!h&&!u&&"draggable"in document.createElement("div"),_e=function(){if(Ce){if(l)return!1;var e=document.createElement("x");return e.style.cssText="pointer-events:auto","auto"===e.style.pointerEvents}}(),Ne=function(e,t){var i=I(e),n=parseInt(i.width)-parseInt(i.paddingLeft)-parseInt(i.paddingRight)-parseInt(i.borderLeftWidth)-parseInt(i.borderRightWidth),o=O(e,0,t),a=O(e,1,t),r=o&&I(o),s=a&&I(a),l=r&&parseInt(r.marginLeft)+parseInt(r.marginRight)+A(o).width,c=s&&parseInt(s.marginLeft)+parseInt(s.marginRight)+A(a).width;if("flex"===i.display)return"column"===i.flexDirection||"column-reverse"===i.flexDirection?"vertical":"horizontal";if("grid"===i.display)return i.gridTemplateColumns.split(" ").length<=1?"vertical":"horizontal";if(o&&r.float&&"none"!==r.float){var p="left"===r.float?"left":"right";return!a||"both"!==s.clear&&s.clear!==p?"horizontal":"vertical"}return o&&("block"===r.display||"flex"===r.display||"table"===r.display||"grid"===r.display||l>=n&&"none"===i[we]||a&&"none"===i[we]&&l+c>n)?"vertical":"horizontal"},Re=function(e){function t(e,i){return function(n,o,a,r){var s=n.options.group.name&&o.options.group.name&&n.options.group.name===o.options.group.name;if(null==e&&(i||s))return!0;if(null==e||!1===e)return!1;if(i&&"clone"===e)return e;if("function"==typeof e)return t(e(n,o,a,r),i)(n,o,a,r);var l=(i?n:o).options.group.name;return!0===e||"string"==typeof e&&e===l||e.join&&e.indexOf(l)>-1}}var i={},o=e.group;o&&"object"==n(o)||(o={name:o}),i.name=o.name,i.checkPull=t(o.pull,!0),i.checkPut=t(o.put),i.revertClone=o.revertClone,e.group=i},Me=function(){!_e&&K&&I(K,"display","none")},Le=function(){!_e&&K&&I(K,"display","")};Ce&&document.addEventListener("click",(function(e){if(ye)return e.preventDefault(),e.stopPropagation&&e.stopPropagation(),e.stopImmediatePropagation&&e.stopImmediatePropagation(),ye=!1,!1}),!0);var De=function(e){if(Y){e=e.touches?e.touches[0]:e;var t=(o=e.clientX,a=e.clientY,Ee.some((function(e){if(!_(e)){var t=A(e),i=e[U].options.emptyInsertThreshold,n=o>=t.left-i&&o<=t.right+i,s=a>=t.top-i&&a<=t.bottom+i;return i&&n&&s?r=e:void 0}})),r);if(t){var i={};for(var n in e)e.hasOwnProperty(n)&&(i[n]=e[n]);i.target=i.rootEl=t,i.preventDefault=void 0,i.stopPropagation=void 0,t[U]._onDragOver(i)}}var o,a,r},He=function(e){Y&&Y.parentNode[U]._isOutsideThisEl(e.target)};function ze(e,t){if(!e||!e.nodeType||1!==e.nodeType)throw"Sortable: `el` must be an HTMLElement, not ".concat({}.toString.call(e));this.el=e,this.options=t=a({},t),e[U]=this;var i,n,o={group:null,sort:!0,disabled:!1,store:null,handle:null,draggable:/^[uo]l$/i.test(e.nodeName)?">li":">*",swapThreshold:1,invertSwap:!1,invertedSwapThreshold:null,removeCloneOnHide:!0,direction:function(){return Ne(e,this.options)},ghostClass:"sortable-ghost",chosenClass:"sortable-chosen",dragClass:"sortable-drag",ignore:"a, img",filter:null,preventOnFilter:!0,animation:0,easing:null,setData:function(e,t){e.setData("Text",t.textContent)},dropBubble:!1,dragoverBubble:!1,dataIdAttr:"data-id",delay:0,delayOnTouchOnly:!1,touchStartThreshold:(Number.parseInt?Number:window).parseInt(window.devicePixelRatio,10)||1,forceFallback:!1,fallbackClass:"sortable-fallback",fallbackOnBody:!1,fallbackTolerance:0,fallbackOffset:{x:0,y:0},supportPointer:!1!==ze.supportPointer&&"PointerEvent"in window,emptyInsertThreshold:5};for(var s in W.initializePlugins(this,e,o),o)!(s in t)&&(t[s]=o[s]);for(var l in Re(t),this)"_"===l.charAt(0)&&"function"==typeof this[l]&&(this[l]=this[l].bind(this));this.nativeDraggable=!t.forceFallback&&Oe,this.nativeDraggable&&(this.options.touchStartThreshold=1),t.supportPointer?m(e,"pointerdown",this._onTapStart):(m(e,"mousedown",this._onTapStart),m(e,"touchstart",this._onTapStart)),this.nativeDraggable&&(m(e,"dragover",this),m(e,"dragenter",this)),Ee.push(this.el),t.store&&t.store.get&&this.sort(t.store.get(this)||[]),a(this,(n=[],{captureAnimationState:function(){n=[],this.options.animation&&[].slice.call(this.el.children).forEach((function(e){if("none"!==I(e,"display")&&e!==ze.ghost){n.push({target:e,rect:A(e)});var t=r({},n[n.length-1].rect);if(e.thisAnimationDuration){var i=T(e,!0);i&&(t.top-=i.f,t.left-=i.e)}e.fromRect=t}}))},addAnimationState:function(e){n.push(e)},removeAnimationState:function(e){n.splice(function(e,t){for(var i in e)if(e.hasOwnProperty(i))for(var n in t)if(t.hasOwnProperty(n)&&t[n]===e[i][n])return Number(i);return-1}(n,{target:e}),1)},animateAll:function(e){var t=this;if(!this.options.animation)return clearTimeout(i),void("function"==typeof e&&e());var o=!1,a=0;n.forEach((function(e){var i=0,n=e.target,r=n.fromRect,s=A(n),l=n.prevFromRect,c=n.prevToRect,p=e.rect,d=T(n,!0);d&&(s.top-=d.f,s.left-=d.e),n.toRect=s,n.thisAnimationDuration&&L(l,s)&&!L(r,s)&&(p.top-s.top)/(p.left-s.left)==(r.top-s.top)/(r.left-s.left)&&(i=function(e,t,i,n){return Math.sqrt(Math.pow(t.top-e.top,2)+Math.pow(t.left-e.left,2))/Math.sqrt(Math.pow(t.top-i.top,2)+Math.pow(t.left-i.left,2))*n.animation}(p,l,c,t.options)),L(s,r)||(n.prevFromRect=r,n.prevToRect=s,i||(i=t.options.animation),t.animate(n,p,s,i)),i&&(o=!0,a=Math.max(a,i),clearTimeout(n.animationResetTimer),n.animationResetTimer=setTimeout((function(){n.animationTime=0,n.prevFromRect=null,n.fromRect=null,n.prevToRect=null,n.thisAnimationDuration=null}),i),n.thisAnimationDuration=i)})),clearTimeout(i),o?i=setTimeout((function(){"function"==typeof e&&e()}),a):"function"==typeof e&&e(),n=[]},animate:function(e,t,i,n){if(n){I(e,"transition",""),I(e,"transform","");var o=T(this.el),a=o&&o.a,r=o&&o.d,s=(t.left-i.left)/(a||1),l=(t.top-i.top)/(r||1);e.animatingX=!!s,e.animatingY=!!l,I(e,"transform","translate3d("+s+"px,"+l+"px,0)"),function(e){e.offsetWidth}(e),I(e,"transition","transform "+n+"ms"+(this.options.easing?" "+this.options.easing:"")),I(e,"transform","translate3d(0,0,0)"),"number"==typeof e.animated&&clearTimeout(e.animated),e.animated=setTimeout((function(){I(e,"transition",""),I(e,"transform",""),e.animated=!1,e.animatingX=!1,e.animatingY=!1}),n)}}}))}function Pe(e,t,i,n,o,a,r,s){var p,d,u=e[U],h=u.options.onMove;return!window.CustomEvent||l||c?(p=document.createEvent("Event")).initEvent("move",!0,!0):p=new CustomEvent("move",{bubbles:!0,cancelable:!0}),p.to=t,p.from=e,p.dragged=i,p.draggedRect=n,p.related=o||t,p.relatedRect=a||A(t),p.willInsertAfter=s,p.originalEvent=r,e.dispatchEvent(p),h&&(d=h.call(u,p,r)),d}function Fe(e){e.draggable=!1}function Ue(){Te=!1}function Be(e){for(var t=e.tagName+e.className+e.src+e.href+e.textContent,i=t.length,n=0;i--;)n+=t.charCodeAt(i);return n.toString(36)}function Ve(e){return setTimeout(e,0)}function We(e){return clearTimeout(e)}ze.prototype={constructor:ze,_isOutsideThisEl:function(e){this.el.contains(e)||e===this.el||(fe=null)},_getDirection:function(e,t){return"function"==typeof this.options.direction?this.options.direction.call(this,e,t,Y):this.options.direction},_onTapStart:function(e){if(e.cancelable){var t=this,i=this.el,n=this.options,o=n.preventOnFilter,a=e.type,r=e.touches&&e.touches[0]||e.pointerType&&"touch"===e.pointerType&&e,s=(r||e).target,l=e.target.shadowRoot&&(e.path&&e.path[0]||e.composedPath&&e.composedPath()[0])||s,c=n.filter;if(function(e){Se.length=0;for(var t=e.getElementsByTagName("input"),i=t.length;i--;){var n=t[i];n.checked&&Se.push(n)}}(i),!Y&&!(/mousedown|pointerdown/.test(a)&&0!==e.button||n.disabled||l.isContentEditable||(s=y(s,n.draggable,i,!1))&&s.animated||J===s)){if(te=N(s),ne=N(s,n.draggable),"function"==typeof c){if(c.call(this,e,s,this))return $({sortable:t,rootEl:l,name:"filter",targetEl:s,toEl:i,fromEl:i}),G("filter",t,{evt:e}),void(o&&e.cancelable&&e.preventDefault())}else if(c&&(c=c.split(",").some((function(n){if(n=y(l,n.trim(),i,!1))return $({sortable:t,rootEl:n,name:"filter",targetEl:s,fromEl:i,toEl:i}),G("filter",t,{evt:e}),!0}))))return void(o&&e.cancelable&&e.preventDefault());n.handle&&!y(l,n.handle,i,!1)||this._prepareDragStart(e,r,s)}}},_prepareDragStart:function(e,t,i){var n,o=this,a=o.el,r=o.options,s=a.ownerDocument;if(i&&!Y&&i.parentNode===a){var d=A(i);if(X=a,q=(Y=i).parentNode,Q=Y.nextSibling,J=i,ae=r.group,ze.dragged=Y,se={target:Y,clientX:(t||e).clientX,clientY:(t||e).clientY},de=se.clientX-d.left,ue=se.clientY-d.top,this._lastX=(t||e).clientX,this._lastY=(t||e).clientY,Y.style["will-change"]="all",n=function(){G("delayEnded",o,{evt:e}),ze.eventCanceled?o._onDrop():(o._disableDelayedDragEvents(),!p&&o.nativeDraggable&&(Y.draggable=!0),o._triggerDragStart(e,t),$({sortable:o,name:"choose",originalEvent:e}),k(Y,r.chosenClass,!0))},r.ignore.split(",").forEach((function(e){S(Y,e.trim(),Fe)})),m(s,"dragover",De),m(s,"mousemove",De),m(s,"touchmove",De),m(s,"mouseup",o._onDrop),m(s,"touchend",o._onDrop),m(s,"touchcancel",o._onDrop),p&&this.nativeDraggable&&(this.options.touchStartThreshold=4,Y.draggable=!0),G("delayStart",this,{evt:e}),!r.delay||r.delayOnTouchOnly&&!t||this.nativeDraggable&&(c||l))n();else{if(ze.eventCanceled)return void this._onDrop();m(s,"mouseup",o._disableDelayedDrag),m(s,"touchend",o._disableDelayedDrag),m(s,"touchcancel",o._disableDelayedDrag),m(s,"mousemove",o._delayedDragTouchMoveHandler),m(s,"touchmove",o._delayedDragTouchMoveHandler),r.supportPointer&&m(s,"pointermove",o._delayedDragTouchMoveHandler),o._dragStartTimer=setTimeout(n,r.delay)}}},_delayedDragTouchMoveHandler:function(e){var t=e.touches?e.touches[0]:e;Math.max(Math.abs(t.clientX-this._lastX),Math.abs(t.clientY-this._lastY))>=Math.floor(this.options.touchStartThreshold/(this.nativeDraggable&&window.devicePixelRatio||1))&&this._disableDelayedDrag()},_disableDelayedDrag:function(){Y&&Fe(Y),clearTimeout(this._dragStartTimer),this._disableDelayedDragEvents()},_disableDelayedDragEvents:function(){var e=this.el.ownerDocument;g(e,"mouseup",this._disableDelayedDrag),g(e,"touchend",this._disableDelayedDrag),g(e,"touchcancel",this._disableDelayedDrag),g(e,"mousemove",this._delayedDragTouchMoveHandler),g(e,"touchmove",this._delayedDragTouchMoveHandler),g(e,"pointermove",this._delayedDragTouchMoveHandler)},_triggerDragStart:function(e,t){t=t||"touch"==e.pointerType&&e,!this.nativeDraggable||t?this.options.supportPointer?m(document,"pointermove",this._onTouchMove):m(document,t?"touchmove":"mousemove",this._onTouchMove):(m(Y,"dragend",this),m(X,"dragstart",this._onDragStart));try{document.selection?Ve((function(){document.selection.empty()})):window.getSelection().removeAllRanges()}catch(e){}},_dragStarted:function(e,t){if(ve=!1,X&&Y){G("dragStarted",this,{evt:t}),this.nativeDraggable&&m(document,"dragover",He);var i=this.options;!e&&k(Y,i.dragClass,!1),k(Y,i.ghostClass,!0),ze.active=this,e&&this._appendGhost(),$({sortable:this,name:"start",originalEvent:t})}else this._nulling()},_emulateDragOver:function(){if(le){this._lastX=le.clientX,this._lastY=le.clientY,Me();for(var e=document.elementFromPoint(le.clientX,le.clientY),t=e;e&&e.shadowRoot&&(e=e.shadowRoot.elementFromPoint(le.clientX,le.clientY))!==t;)t=e;if(Y.parentNode[U]._isOutsideThisEl(e),t)do{if(t[U]&&t[U]._onDragOver({clientX:le.clientX,clientY:le.clientY,target:e,rootEl:t})&&!this.options.dragoverBubble)break;e=t}while(t=t.parentNode);Le()}},_onTouchMove:function(e){if(se){var t=this.options,i=t.fallbackTolerance,n=t.fallbackOffset,o=e.touches?e.touches[0]:e,a=K&&T(K,!0),r=K&&a&&a.a,s=K&&a&&a.d,l=Ae&&be&&R(be),c=(o.clientX-se.clientX+n.x)/(r||1)+(l?l[0]-Ie[0]:0)/(r||1),p=(o.clientY-se.clientY+n.y)/(s||1)+(l?l[1]-Ie[1]:0)/(s||1);if(!ze.active&&!ve){if(i&&Math.max(Math.abs(o.clientX-this._lastX),Math.abs(o.clientY-this._lastY))n.right+10||e.clientX<=n.right&&e.clientY>n.bottom&&e.clientX>=n.left:e.clientX>n.right&&e.clientY>n.top||e.clientX<=n.right&&e.clientY>n.bottom+10}(e,o,this)&&!g.animated){if(g===Y)return F(!1);if(g&&a===e.target&&(s=g),s&&(i=A(s)),!1!==Pe(X,a,Y,t,s,i,e,!!s))return P(),a.appendChild(Y),q=a,B(),F(!0)}else if(s.parentNode===a){i=A(s);var b,v,E,x=Y.parentNode!==a,T=!function(e,t,i){var n=i?e.left:e.top,o=i?e.right:e.bottom,a=i?e.width:e.height,r=i?t.left:t.top,s=i?t.right:t.bottom,l=i?t.width:t.height;return n===r||o===s||n+a/2===r+l/2}(Y.animated&&Y.toRect||t,s.animated&&s.toRect||i,o),S=o?"top":"left",C=w(s,"top","top")||w(Y,"top","top"),O=C?C.scrollTop:void 0;if(fe!==s&&(v=i[S],xe=!1,ke=!T&&l.invertSwap||x),b=function(e,t,i,n,o,a,r,s){var l=n?e.clientY:e.clientX,c=n?i.height:i.width,p=n?i.top:i.left,d=n?i.bottom:i.right,u=!1;if(!r)if(s&&gep+c*a/2:ld-ge)return-me}else if(l>p+c*(1-o)/2&&ld-c*a/2)?l>p+c/2?1:-1:0}(e,s,i,o,T?1:l.swapThreshold,null==l.invertedSwapThreshold?l.swapThreshold:l.invertedSwapThreshold,ke,fe===s),0!==b){var R=N(Y);do{R-=b,E=q.children[R]}while(E&&("none"===I(E,"display")||E===K))}if(0===b||E===s)return F(!1);fe=s,me=b;var M=s.nextElementSibling,L=!1,D=Pe(X,a,Y,t,s,i,e,L=1===b);if(!1!==D)return 1!==D&&-1!==D||(L=1===D),Te=!0,setTimeout(Ue,30),P(),L&&!M?a.appendChild(Y):s.parentNode.insertBefore(Y,L?M:s),C&&H(C,0,O-C.scrollTop),q=Y.parentNode,void 0===v||ke||(ge=Math.abs(v-A(s)[S])),B(),F(!0)}if(a.contains(Y))return F(!1)}return!1}function z(l,c){G(l,f,r({evt:e,isOwner:d,axis:o?"vertical":"horizontal",revert:n,dragRect:t,targetRect:i,canSort:u,fromSortable:h,target:s,completed:F,onMove:function(i,n){return Pe(X,a,Y,t,i,A(i),e,n)},changed:B},c))}function P(){z("dragOverAnimationCapture"),f.captureAnimationState(),f!==h&&h.captureAnimationState()}function F(t){return z("dragOverCompleted",{insertion:t}),t&&(d?p._hideClone():p._showClone(f),f!==h&&(k(Y,re?re.options.ghostClass:p.options.ghostClass,!1),k(Y,l.ghostClass,!0)),re!==f&&f!==ze.active?re=f:f===ze.active&&re&&(re=null),h===f&&(f._ignoreWhileAnimating=s),f.animateAll((function(){z("dragOverAnimationComplete"),f._ignoreWhileAnimating=null})),f!==h&&(h.animateAll(),h._ignoreWhileAnimating=null)),(s===Y&&!Y.animated||s===a&&!s.animated)&&(fe=null),l.dragoverBubble||e.rootEl||s===document||(Y.parentNode[U]._isOutsideThisEl(e.target),!t&&De(e)),!l.dragoverBubble&&e.stopPropagation&&e.stopPropagation(),m=!0}function B(){ie=N(Y),oe=N(Y,l.draggable),$({sortable:f,name:"change",toEl:a,newIndex:ie,newDraggableIndex:oe,originalEvent:e})}},_ignoreWhileAnimating:null,_offMoveEvents:function(){g(document,"mousemove",this._onTouchMove),g(document,"touchmove",this._onTouchMove),g(document,"pointermove",this._onTouchMove),g(document,"dragover",De),g(document,"mousemove",De),g(document,"touchmove",De)},_offUpEvents:function(){var e=this.el.ownerDocument;g(e,"mouseup",this._onDrop),g(e,"touchend",this._onDrop),g(e,"pointerup",this._onDrop),g(e,"touchcancel",this._onDrop),g(document,"selectstart",this)},_onDrop:function(e){var t=this.el,i=this.options;ie=N(Y),oe=N(Y,i.draggable),G("drop",this,{evt:e}),q=Y&&Y.parentNode,ie=N(Y),oe=N(Y,i.draggable),ze.eventCanceled||(ve=!1,ke=!1,xe=!1,clearInterval(this._loopId),clearTimeout(this._dragStartTimer),We(this.cloneId),We(this._dragStartId),this.nativeDraggable&&(g(document,"drop",this),g(t,"dragstart",this._onDragStart)),this._offMoveEvents(),this._offUpEvents(),d&&I(document.body,"user-select",""),I(Y,"transform",""),e&&(he&&(e.cancelable&&e.preventDefault(),!i.dropBubble&&e.stopPropagation()),K&&K.parentNode&&K.parentNode.removeChild(K),(X===q||re&&"clone"!==re.lastPutMode)&&Z&&Z.parentNode&&Z.parentNode.removeChild(Z),Y&&(this.nativeDraggable&&g(Y,"dragend",this),Fe(Y),Y.style["will-change"]="",he&&!ve&&k(Y,re?re.options.ghostClass:this.options.ghostClass,!1),k(Y,this.options.chosenClass,!1),$({sortable:this,name:"unchoose",toEl:q,newIndex:null,newDraggableIndex:null,originalEvent:e}),X!==q?(ie>=0&&($({rootEl:q,name:"add",toEl:q,fromEl:X,originalEvent:e}),$({sortable:this,name:"remove",toEl:q,originalEvent:e}),$({rootEl:q,name:"sort",toEl:q,fromEl:X,originalEvent:e}),$({sortable:this,name:"sort",toEl:q,originalEvent:e})),re&&re.save()):ie!==te&&ie>=0&&($({sortable:this,name:"update",toEl:q,originalEvent:e}),$({sortable:this,name:"sort",toEl:q,originalEvent:e})),ze.active&&(null!=ie&&-1!==ie||(ie=te,oe=ne),$({sortable:this,name:"end",toEl:q,originalEvent:e}),this.save())))),this._nulling()},_nulling:function(){G("nulling",this),X=Y=q=K=Q=Z=J=ee=se=le=he=ie=oe=te=ne=fe=me=re=ae=ze.dragged=ze.ghost=ze.clone=ze.active=null,Se.forEach((function(e){e.checked=!0})),Se.length=ce=pe=0},handleEvent:function(e){switch(e.type){case"drop":case"dragend":this._onDrop(e);break;case"dragenter":case"dragover":Y&&(this._onDragOver(e),function(e){e.dataTransfer&&(e.dataTransfer.dropEffect="move"),e.cancelable&&e.preventDefault()}(e));break;case"selectstart":e.preventDefault()}},toArray:function(){for(var e,t=[],i=this.el.children,n=0,o=i.length,a=this.options;n1&&(dt.forEach((function(e){n.addAnimationState({target:e,rect:ft?A(e):o}),F(e),e.fromRect=o,t.removeAnimationState(e)})),ft=!1,function(e,t){dt.forEach((function(i,n){var o=t.children[i.sortableIndex+(e?Number(n):0)];o?t.insertBefore(i,o):t.appendChild(i)}))}(!this.options.removeCloneOnHide,i))},dragOverCompleted:function(e){var t=e.sortable,i=e.isOwner,n=e.insertion,o=e.activeSortable,a=e.parentEl,r=e.putSortable,s=this.options;if(n){if(i&&o._hideClone(),ht=!1,s.animation&&dt.length>1&&(ft||!i&&!o.options.sort&&!r)){var l=A(lt,!1,!0,!0);dt.forEach((function(e){e!==lt&&(P(e,l),a.appendChild(e))})),ft=!0}if(!i)if(ft||vt(),dt.length>1){var c=pt;o._showClone(t),o.options.animation&&!pt&&c&&ut.forEach((function(e){o.addAnimationState({target:e,rect:ct}),e.fromRect=ct,e.thisAnimationDuration=null}))}else o._showClone(t)}},dragOverAnimationCapture:function(e){var t=e.dragRect,i=e.isOwner,n=e.activeSortable;if(dt.forEach((function(e){e.thisAnimationDuration=null})),n.options.animation&&!i&&n.multiDrag.isMultiDrag){ct=a({},t);var o=T(lt,!0);ct.top-=o.f,ct.left-=o.e}},dragOverAnimationComplete:function(){ft&&(ft=!1,vt())},drop:function(e){var t=e.originalEvent,i=e.rootEl,n=e.parentEl,o=e.sortable,a=e.dispatchSortableEvent,r=e.oldIndex,s=e.putSortable,l=s||this.sortable;if(t){var c=this.options,p=n.children;if(!mt)if(c.multiDragKey&&!this.multiDragKeyDown&&this._deselectMultiDrag(),k(lt,c.selectedClass,!~dt.indexOf(lt)),~dt.indexOf(lt))dt.splice(dt.indexOf(lt),1),rt=null,j({sortable:o,rootEl:i,name:"deselect",targetEl:lt,originalEvt:t});else{if(dt.push(lt),j({sortable:o,rootEl:i,name:"select",targetEl:lt,originalEvt:t}),t.shiftKey&&rt&&o.el.contains(rt)){var d,u,h=N(rt),f=N(lt);if(~h&&~f&&h!==f)for(f>h?(u=h,d=f):(u=f,d=h+1);u1){var m=A(lt),g=N(lt,":not(."+this.options.selectedClass+")");if(!ht&&c.animation&&(lt.thisAnimationDuration=null),l.captureAnimationState(),!ht&&(c.animation&&(lt.fromRect=m,dt.forEach((function(e){if(e.thisAnimationDuration=null,e!==lt){var t=ft?A(e):m;e.fromRect=t,l.addAnimationState({target:e,rect:t})}}))),vt(),dt.forEach((function(e){p[g]?n.insertBefore(e,p[g]):n.appendChild(e),g++})),r===N(lt))){var b=!1;dt.forEach((function(e){e.sortableIndex===N(e)||(b=!0)})),b&&a("update")}dt.forEach((function(e){F(e)})),l.animateAll()}st=l}(i===n||s&&"clone"!==s.lastPutMode)&&ut.forEach((function(e){e.parentNode&&e.parentNode.removeChild(e)}))}},nullingGlobal:function(){this.isMultiDrag=mt=!1,ut.length=0},destroyGlobal:function(){this._deselectMultiDrag(),g(document,"pointerup",this._deselectMultiDrag),g(document,"mouseup",this._deselectMultiDrag),g(document,"touchend",this._deselectMultiDrag),g(document,"keydown",this._checkKeyDown),g(document,"keyup",this._checkKeyUp)},_deselectMultiDrag:function(e){if(!(void 0!==mt&&mt||st!==this.sortable||e&&y(e.target,this.options.draggable,this.sortable.el,!1)||e&&0!==e.button))for(;dt.length;){var t=dt[0];k(t,this.options.selectedClass,!1),dt.shift(),j({sortable:this.sortable,rootEl:this.sortable.el,name:"deselect",targetEl:t,originalEvt:e})}},_checkKeyDown:function(e){e.key===this.options.multiDragKey&&(this.multiDragKeyDown=!0)},_checkKeyUp:function(e){e.key===this.options.multiDragKey&&(this.multiDragKeyDown=!1)}},a(e,{pluginName:"multiDrag",utils:{select:function(e){var t=e.parentNode[U];t&&t.options.multiDrag&&!~dt.indexOf(e)&&(st&&st!==t&&(st.multiDrag._deselectMultiDrag(),st=t),k(e,t.options.selectedClass,!0),dt.push(e))},deselect:function(e){var t=e.parentNode[U],i=dt.indexOf(e);t&&t.options.multiDrag&&~i&&(k(e,t.options.selectedClass,!1),dt.splice(i,1))}},eventProperties:function(){var e,t=this,i=[],n=[];return dt.forEach((function(e){var o;i.push({multiDragElement:e,index:e.sortableIndex}),o=ft&&e!==lt?-1:ft?N(e,":not(."+t.options.selectedClass+")"):N(e),n.push({multiDragElement:e,index:o})})),{items:(e=dt,function(e){if(Array.isArray(e)){for(var t=0,i=new Array(e.length);t1&&(e=e.charAt(0).toUpperCase()+e.substr(1)),e}}})}function bt(e,t){ut.forEach((function(i,n){var o=t.children[i.sortableIndex+(e?Number(n):0)];o?t.insertBefore(i,o):t.appendChild(i)}))}function vt(){dt.forEach((function(e){e!==lt&&e.parentNode&&e.parentNode.removeChild(e)}))}ze.mount(new function(){function e(){for(var e in this.defaults={scroll:!0,scrollSensitivity:30,scrollSpeed:10,bubbleScroll:!0},this)"_"===e.charAt(0)&&"function"==typeof this[e]&&(this[e]=this[e].bind(this))}return e.prototype={dragStarted:function(e){var t=e.originalEvent;this.sortable.nativeDraggable?m(document,"dragover",this._handleAutoScroll):this.options.supportPointer?m(document,"pointermove",this._handleFallbackAutoScroll):t.touches?m(document,"touchmove",this._handleFallbackAutoScroll):m(document,"mousemove",this._handleFallbackAutoScroll)},dragOverCompleted:function(e){var t=e.originalEvent;this.options.dragOverBubble||t.rootEl||this._handleAutoScroll(t)},drop:function(){this.sortable.nativeDraggable?g(document,"dragover",this._handleAutoScroll):(g(document,"pointermove",this._handleFallbackAutoScroll),g(document,"touchmove",this._handleFallbackAutoScroll),g(document,"mousemove",this._handleFallbackAutoScroll)),Ze(),Je(),clearTimeout(E),E=void 0},nulling:function(){qe=Ge=je=Qe=Ke=$e=Ye=null,Xe.length=0},_handleFallbackAutoScroll:function(e){this._handleAutoScroll(e,!0)},_handleAutoScroll:function(e,t){var i=this,n=(e.touches?e.touches[0]:e).clientX,o=(e.touches?e.touches[0]:e).clientY,a=document.elementFromPoint(n,o);if(qe=e,t||c||l||d){tt(e,this.options,a,t);var r=M(a,!0);!Qe||Ke&&n===$e&&o===Ye||(Ke&&Ze(),Ke=setInterval((function(){var a=M(document.elementFromPoint(n,o),!0);a!==r&&(r=a,Je()),tt(e,i.options,a,t)}),10),$e=n,Ye=o)}else{if(!this.options.bubbleScroll||M(a,!0)===C())return void Je();tt(e,this.options,M(a,!1),!1)}}},a(e,{pluginName:"scroll",initializeByDefault:!0})}),ze.mount(ot,nt);const yt=ze},946:e=>{"use strict";var t=[];function i(e){for(var i=-1,n=0;n{"use strict";var t={};e.exports=function(e,i){var n=function(e){if(void 0===t[e]){var i=document.querySelector(e);if(window.HTMLIFrameElement&&i instanceof window.HTMLIFrameElement)try{i=i.contentDocument.head}catch(e){i=null}t[e]=i}return t[e]}(e);if(!n)throw new Error("Couldn't find a style target. This probably means that the value for the 'insert' parameter is invalid.");n.appendChild(i)}},430:e=>{"use strict";e.exports=function(e){var t=document.createElement("style");return e.setAttributes(t,e.attributes),e.insert(t,e.options),t}},714:(e,t,i)=>{"use strict";e.exports=function(e){var t=i.nc;t&&e.setAttribute("nonce",t)}},571:e=>{"use strict";e.exports=function(e){if("undefined"==typeof document)return{update:function(){},remove:function(){}};var t=e.insertStyleElement(e);return{update:function(i){!function(e,t,i){var n="";i.supports&&(n+="@supports (".concat(i.supports,") {")),i.media&&(n+="@media ".concat(i.media," {"));var o=void 0!==i.layer;o&&(n+="@layer".concat(i.layer.length>0?" ".concat(i.layer):""," {")),n+=i.css,o&&(n+="}"),i.media&&(n+="}"),i.supports&&(n+="}");var a=i.sourceMap;a&&"undefined"!=typeof btoa&&(n+="\n/*# sourceMappingURL=data:application/json;base64,".concat(btoa(unescape(encodeURIComponent(JSON.stringify(a))))," */")),t.styleTagTransform(n,e,t.options)}(t,e,i)},remove:function(){!function(e){if(null===e.parentNode)return!1;e.parentNode.removeChild(e)}(t)}}}},307:e=>{"use strict";e.exports=function(e,t){if(t.styleSheet)t.styleSheet.cssText=e;else{for(;t.firstChild;)t.removeChild(t.firstChild);t.appendChild(document.createTextNode(e))}}},102:function(e,t,i){var n;"undefined"!=typeof self&&self,n=function(e){return function(e){var t={};function i(n){if(t[n])return t[n].exports;var o=t[n]={i:n,l:!1,exports:{}};return e[n].call(o.exports,o,o.exports,i),o.l=!0,o.exports}return i.m=e,i.c=t,i.d=function(e,t,n){i.o(e,t)||Object.defineProperty(e,t,{enumerable:!0,get:n})},i.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},i.t=function(e,t){if(1&t&&(e=i(e)),8&t)return e;if(4&t&&"object"==typeof e&&e&&e.__esModule)return e;var n=Object.create(null);if(i.r(n),Object.defineProperty(n,"default",{enumerable:!0,value:e}),2&t&&"string"!=typeof e)for(var o in e)i.d(n,o,function(t){return e[t]}.bind(null,o));return n},i.n=function(e){var t=e&&e.__esModule?function(){return e.default}:function(){return e};return i.d(t,"a",t),t},i.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)},i.p="",i(i.s="fb15")}({"01f9":function(e,t,i){"use strict";var n=i("2d00"),o=i("5ca1"),a=i("2aba"),r=i("32e9"),s=i("84f2"),l=i("41a0"),c=i("7f20"),p=i("38fd"),d=i("2b4c")("iterator"),u=!([].keys&&"next"in[].keys()),h="keys",f="values",m=function(){return this};e.exports=function(e,t,i,g,b,v,y){l(i,t,g);var E,x,k,I=function(e){if(!u&&e in A)return A[e];switch(e){case h:case f:return function(){return new i(this,e)}}return function(){return new i(this,e)}},T=t+" Iterator",S=b==f,C=!1,A=e.prototype,w=A[d]||A["@@iterator"]||b&&A[b],O=w||I(b),_=b?S?I("entries"):O:void 0,N="Array"==t&&A.entries||w;if(N&&(k=p(N.call(new e)))!==Object.prototype&&k.next&&(c(k,T,!0),n||"function"==typeof k[d]||r(k,d,m)),S&&w&&w.name!==f&&(C=!0,O=function(){return w.call(this)}),n&&!y||!u&&!C&&A[d]||r(A,d,O),s[t]=O,s[T]=m,b)if(E={values:S?O:I(f),keys:v?O:I(h),entries:_},y)for(x in E)x in A||a(A,x,E[x]);else o(o.P+o.F*(u||C),t,E);return E}},"02f4":function(e,t,i){var n=i("4588"),o=i("be13");e.exports=function(e){return function(t,i){var a,r,s=String(o(t)),l=n(i),c=s.length;return l<0||l>=c?e?"":void 0:(a=s.charCodeAt(l))<55296||a>56319||l+1===c||(r=s.charCodeAt(l+1))<56320||r>57343?e?s.charAt(l):a:e?s.slice(l,l+2):r-56320+(a-55296<<10)+65536}}},"0390":function(e,t,i){"use strict";var n=i("02f4")(!0);e.exports=function(e,t,i){return t+(i?n(e,t).length:1)}},"0bfb":function(e,t,i){"use strict";var n=i("cb7c");e.exports=function(){var e=n(this),t="";return e.global&&(t+="g"),e.ignoreCase&&(t+="i"),e.multiline&&(t+="m"),e.unicode&&(t+="u"),e.sticky&&(t+="y"),t}},"0d58":function(e,t,i){var n=i("ce10"),o=i("e11e");e.exports=Object.keys||function(e){return n(e,o)}},1495:function(e,t,i){var n=i("86cc"),o=i("cb7c"),a=i("0d58");e.exports=i("9e1e")?Object.defineProperties:function(e,t){o(e);for(var i,r=a(t),s=r.length,l=0;s>l;)n.f(e,i=r[l++],t[i]);return e}},"214f":function(e,t,i){"use strict";i("b0c5");var n=i("2aba"),o=i("32e9"),a=i("79e5"),r=i("be13"),s=i("2b4c"),l=i("520a"),c=s("species"),p=!a((function(){var e=/./;return e.exec=function(){var e=[];return e.groups={a:"7"},e},"7"!=="".replace(e,"$")})),d=function(){var e=/(?:)/,t=e.exec;e.exec=function(){return t.apply(this,arguments)};var i="ab".split(e);return 2===i.length&&"a"===i[0]&&"b"===i[1]}();e.exports=function(e,t,i){var u=s(e),h=!a((function(){var t={};return t[u]=function(){return 7},7!=""[e](t)})),f=h?!a((function(){var t=!1,i=/a/;return i.exec=function(){return t=!0,null},"split"===e&&(i.constructor={},i.constructor[c]=function(){return i}),i[u](""),!t})):void 0;if(!h||!f||"replace"===e&&!p||"split"===e&&!d){var m=/./[u],g=i(r,u,""[e],(function(e,t,i,n,o){return t.exec===l?h&&!o?{done:!0,value:m.call(t,i,n)}:{done:!0,value:e.call(i,t,n)}:{done:!1}})),b=g[0],v=g[1];n(String.prototype,e,b),o(RegExp.prototype,u,2==t?function(e,t){return v.call(e,this,t)}:function(e){return v.call(e,this)})}}},"230e":function(e,t,i){var n=i("d3f4"),o=i("7726").document,a=n(o)&&n(o.createElement);e.exports=function(e){return a?o.createElement(e):{}}},"23c6":function(e,t,i){var n=i("2d95"),o=i("2b4c")("toStringTag"),a="Arguments"==n(function(){return arguments}());e.exports=function(e){var t,i,r;return void 0===e?"Undefined":null===e?"Null":"string"==typeof(i=function(e,t){try{return e[t]}catch(e){}}(t=Object(e),o))?i:a?n(t):"Object"==(r=n(t))&&"function"==typeof t.callee?"Arguments":r}},2621:function(e,t){t.f=Object.getOwnPropertySymbols},"2aba":function(e,t,i){var n=i("7726"),o=i("32e9"),a=i("69a8"),r=i("ca5a")("src"),s=i("fa5b"),l="toString",c=(""+s).split(l);i("8378").inspectSource=function(e){return s.call(e)},(e.exports=function(e,t,i,s){var l="function"==typeof i;l&&(a(i,"name")||o(i,"name",t)),e[t]!==i&&(l&&(a(i,r)||o(i,r,e[t]?""+e[t]:c.join(String(t)))),e===n?e[t]=i:s?e[t]?e[t]=i:o(e,t,i):(delete e[t],o(e,t,i)))})(Function.prototype,l,(function(){return"function"==typeof this&&this[r]||s.call(this)}))},"2aeb":function(e,t,i){var n=i("cb7c"),o=i("1495"),a=i("e11e"),r=i("613b")("IE_PROTO"),s=function(){},l="prototype",c=function(){var e,t=i("230e")("iframe"),n=a.length;for(t.style.display="none",i("fab2").appendChild(t),t.src="javascript:",(e=t.contentWindow.document).open(),e.write(" diff --git a/src/frontend/devops-codelib/src/components/AtomFormField/VuexInput.vue b/src/frontend/devops-codelib/src/components/AtomFormField/VuexInput.vue index 58ff1bba9f7..a82a7cf0838 100644 --- a/src/frontend/devops-codelib/src/components/AtomFormField/VuexInput.vue +++ b/src/frontend/devops-codelib/src/components/AtomFormField/VuexInput.vue @@ -26,8 +26,7 @@ handleInput (e, isBlur = false) { const { value, name } = e.target const trimVal = isBlur ? value.trim() : value - - if (trimVal !== String(this.value).trim() || trimVal !== '') { + if (trimVal !== this.value || trimVal !== '') { this.$emit('input', trimVal) this.handleChange(name, trimVal) } @@ -40,7 +39,18 @@ const { inputType, value, name, handleInput, handleBlur, title, placeholder, maxLength } = this return ( - + ) } } diff --git a/src/frontend/devops-codelib/src/components/CodeLibDetail/basic-setting.vue b/src/frontend/devops-codelib/src/components/CodeLibDetail/basic-setting.vue index b661c94dcb4..409786ca5fc 100644 --- a/src/frontend/devops-codelib/src/components/CodeLibDetail/basic-setting.vue +++ b/src/frontend/devops-codelib/src/components/CodeLibDetail/basic-setting.vue @@ -248,7 +248,7 @@ {{ $t('codelib.关闭 PAC 模式') }} -

{{ $t('codelib.检测到默认分支仍存在ci 文件目录，关闭 PAC 模式后该目录下的文件修改将') }}

+

{{ $t('codelib.检测到默认分支仍存在ci 文件目录。') }}

{{ $t('codelib.请先将目录') }} {{ $t('codelib.改名或删除') }} diff --git a/src/frontend/devops-codelib/src/components/CodeLibDetail/index.vue b/src/frontend/devops-codelib/src/components/CodeLibDetail/index.vue index 8f039028e74..8ced4b07686 100644 --- a/src/frontend/devops-codelib/src/components/CodeLibDetail/index.vue +++ b/src/frontend/devops-codelib/src/components/CodeLibDetail/index.vue @@ -267,6 +267,9 @@ userId () { return this.$route.query.userId || '' }, + resetType () { + return this.$route.query.resetType || '' + }, scmType () { return this.$route.query.scmType || '' }, @@ -367,7 +370,7 @@ this.errorCode = e.httpStatus || 404 }).finally(() => { if (this.userId) { - this.isLoading = loading + this.isLoading = this.resetType ? loading : false } else { this.isLoading = false } diff --git a/src/frontend/devops-codelib/src/components/CodeLibDetail/trigger.vue b/src/frontend/devops-codelib/src/components/CodeLibDetail/trigger.vue index 90f5e301988..7c6c1793d74 100644 --- a/src/frontend/devops-codelib/src/components/CodeLibDetail/trigger.vue +++ b/src/frontend/devops-codelib/src/components/CodeLibDetail/trigger.vue @@ -22,7 +22,7 @@ diff --git a/src/frontend/devops-codelib/src/components/CodeLibDialog/mixin.js b/src/frontend/devops-codelib/src/components/CodeLibDialog/mixin.js index 4baf9e975a1..82e19e64c30 100644 --- a/src/frontend/devops-codelib/src/components/CodeLibDialog/mixin.js +++ b/src/frontend/devops-codelib/src/components/CodeLibDialog/mixin.js @@ -255,6 +255,12 @@ export default { param.aliasName = alias || this.codelib.aliasName this.updateCodelib(param) + }, + + showCodelibDialog (val) { + if (!val) { + this.pacProjectName = '' + } } }, diff --git a/src/frontend/devops-codelib/src/components/LinkCodeLib/index.vue b/src/frontend/devops-codelib/src/components/LinkCodeLib/index.vue index 9484811be5d..b71df2fa6be 100755 --- a/src/frontend/devops-codelib/src/components/LinkCodeLib/index.vue +++ b/src/frontend/devops-codelib/src/components/LinkCodeLib/index.vue @@ -1,19 +1,26 @@ diff --git a/src/frontend/devops-codelib/src/views/Index.vue b/src/frontend/devops-codelib/src/views/Index.vue index 3c6d177845a..2be5646b827 100755 --- a/src/frontend/devops-codelib/src/views/Index.vue +++ b/src/frontend/devops-codelib/src/views/Index.vue @@ -2,7 +2,10 @@

+ diff --git a/src/frontend/devops-pipeline/package.json b/src/frontend/devops-pipeline/package.json index 3dfc2706df6..8de926c56f0 100755 --- a/src/frontend/devops-pipeline/package.json +++ b/src/frontend/devops-pipeline/package.json @@ -14,10 +14,10 @@ "public:external": "cross-env NODE_ENV=external npm run public --" }, "dependencies": { - "@blueking/log": "2.1.4", + "@blueking/log": "2.1.5", "@blueking/search-select": "0.0.1-beta.2", "@icon-cool/bk-icon-devops": "^0.2.1", - "axios": "^0.27.2", + "axios": "^0.28.0", "bk-magic-vue": "2.5.9-beta.9", "bk-permission": "^0.0.28", "bkui-pipeline": "0.1.34", diff --git a/src/frontend/devops-pipeline/src/App.vue b/src/frontend/devops-pipeline/src/App.vue index d4bdeed2414..1da3628a02f 100755 --- a/src/frontend/devops-pipeline/src/App.vue +++ b/src/frontend/devops-pipeline/src/App.vue @@ -1,5 +1,8 @@
$pipelineNameTitle：$pipelineName$pipelineLinkElement触发方式：$triggerType质量红线：