From 48aa0d9405b2b044c75243c028f5afe6232d4451 Mon Sep 17 00:00:00 2001 From: guohelu <19503896967@163.com> Date: Wed, 11 Dec 2024 21:52:52 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E4=BF=AE=E6=94=B9=E6=9D=83=E9=99=90?= =?UTF-8?q?=E9=97=AE=E9=A2=98=20#7626?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/default.py | 2 + env.py | 2 + .../commands/data/api-resources.yml | 2 +- .../copy_template_across_project.py | 5 +- gcloud/contrib/template_market/admin.py | 8 +- .../migrations/0001_initial.py | 7 +- gcloud/contrib/template_market/models.py | 21 +-- gcloud/contrib/template_market/permission.py | 29 +++- gcloud/contrib/template_market/serializers.py | 72 +++++++- gcloud/contrib/template_market/utils.py | 39 +++++ gcloud/contrib/template_market/viewsets.py | 157 +++++++++++------- 11 files changed, 245 insertions(+), 99 deletions(-) create mode 100644 gcloud/contrib/template_market/utils.py diff --git a/config/default.py b/config/default.py index be7bbd0ad..c771f2f3f 100644 --- a/config/default.py +++ b/config/default.py @@ -890,3 +890,5 @@ def check_engine_admin_permission(request, *args, **kwargs): ENABLE_TEMPLATE_MARKET = env.ENABLE_TEMPLATE_MARKET # 流程商店 API 地址 TEMPLATE_MARKET_API_URL = env.TEMPLATE_MARKET_API_URL +# 共享流程最大数量 +MAX_NUMBER_SHARED_PROCESSES = env.MAX_NUMBER_SHARED_PROCESSES diff --git a/env.py b/env.py index 2a4e1546e..cab2ee11d 100644 --- a/env.py +++ b/env.py @@ -158,3 +158,5 @@ ENABLE_TEMPLATE_MARKET = False if os.getenv("ENABLE_TEMPLATE_MARKET") is None else True # 流程商店 API 地址 TEMPLATE_MARKET_API_URL = os.getenv("TEMPLATE_MARKET_API_URL", "") +# 共享流程最大数量 +MAX_NUMBER_SHARED_PROCESSES = int(os.getenv("MAX_NUMBER_SHARED_PROCESSES", 10)) diff --git a/gcloud/apigw/management/commands/data/api-resources.yml b/gcloud/apigw/management/commands/data/api-resources.yml index d76ef16d2..cdabd3892 100644 --- a/gcloud/apigw/management/commands/data/api-resources.yml +++ b/gcloud/apigw/management/commands/data/api-resources.yml @@ -767,7 +767,7 @@ paths: default: description: '' x-bk-apigateway-resource: - isPublic: true + isPublic: false allowApplyPermission: true matchSubpath: false backend: diff --git a/gcloud/apigw/validators/copy_template_across_project.py b/gcloud/apigw/validators/copy_template_across_project.py index 27eaa1cc1..53d609477 100644 --- a/gcloud/apigw/validators/copy_template_across_project.py +++ b/gcloud/apigw/validators/copy_template_across_project.py @@ -22,7 +22,8 @@ def validate(self, request, *args, **kwargs): if not valid: return valid, err - if not json.loads(request.body).get("new_project_id") or not json.loads(request.body).get("template_id"): - return False, "new_project_id and template_id is required" + data = json.loads(request.body) + if not data.get("new_project_id") or not data.get("template_id"): + return False, "new_project_id and template_id are required" return True, "" diff --git a/gcloud/contrib/template_market/admin.py b/gcloud/contrib/template_market/admin.py index 7a89f6a28..9083b78bd 100644 --- a/gcloud/contrib/template_market/admin.py +++ b/gcloud/contrib/template_market/admin.py @@ -17,7 +17,7 @@ @admin.register(models.TemplateSharedRecord) -class TemplateMarketAdmin(admin.ModelAdmin): - list_display = ["project_id", "template_id", "scene_instance_id", "creator", "create_at", "extra_info"] - list_filter = ["project_id", "creator", "create_at"] - search_fields = ["project_id", "template_id", "scene_instance_id", "creator"] +class TemplateSharedRecordAdmin(admin.ModelAdmin): + list_display = ["scene_shared_id", "project_id", "templates", "creator", "create_at", "update_at", "extra_info"] + list_filter = ["project_id", "creator", "create_at", "update_at"] + search_fields = ["scene_shared_id", "project_id", "creator"] diff --git a/gcloud/contrib/template_market/migrations/0001_initial.py b/gcloud/contrib/template_market/migrations/0001_initial.py index e9fb9abbe..50b0871bd 100644 --- a/gcloud/contrib/template_market/migrations/0001_initial.py +++ b/gcloud/contrib/template_market/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 3.2.15 on 2024-12-10 02:27 +# Generated by Django 3.2.15 on 2024-12-11 13:46 from django.db import migrations, models @@ -14,11 +14,12 @@ class Migration(migrations.Migration): name="TemplateSharedRecord", fields=[ ("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("scene_shared_id", models.IntegerField(help_text="共享实例id", verbose_name="共享实例id")), ("project_id", models.IntegerField(default=-1, help_text="项目 ID", verbose_name="项目 ID")), - ("template_id", models.IntegerField(help_text="模版 ID", verbose_name="模版 ID")), - ("scene_instance_id", models.IntegerField(db_index=True, help_text="场景实例 ID", verbose_name="场景实例 ID")), + ("templates", models.JSONField(help_text="模板 ID 列表", verbose_name="模板 ID 列表")), ("creator", models.CharField(default="", max_length=32, verbose_name="创建者")), ("create_at", models.DateTimeField(auto_now_add=True, verbose_name="创建时间")), + ("update_at", models.DateTimeField(auto_now=True, verbose_name="更新时间")), ("extra_info", models.JSONField(blank=True, null=True, verbose_name="额外信息")), ], options={ diff --git a/gcloud/contrib/template_market/models.py b/gcloud/contrib/template_market/models.py index 9ca074161..f16eaf7ab 100644 --- a/gcloud/contrib/template_market/models.py +++ b/gcloud/contrib/template_market/models.py @@ -11,34 +11,19 @@ specific language governing permissions and limitations under the License. """ - from django.db import models from django.utils.translation import ugettext_lazy as _ class TemplateSharedRecord(models.Model): + scene_shared_id = models.IntegerField(_("共享实例id"), help_text="共享实例id") project_id = models.IntegerField(_("项目 ID"), default=-1, help_text="项目 ID") - template_id = models.IntegerField(_("模版 ID"), help_text="模版 ID") - scene_instance_id = models.IntegerField(_("场景实例 ID"), db_index=True, help_text="场景实例 ID") + templates = models.JSONField(_("模板 ID 列表"), help_text="模板 ID 列表") creator = models.CharField(_("创建者"), max_length=32, default="") create_at = models.DateTimeField(_("创建时间"), auto_now_add=True) + update_at = models.DateTimeField(verbose_name=_("更新时间"), auto_now=True) extra_info = models.JSONField(_("额外信息"), blank=True, null=True) class Meta: verbose_name = _("模板共享记录 TemplateSharedRecord") verbose_name_plural = _("模板共享记录 TemplateSharedRecord") - - @classmethod - def create(cls, project_id, template_id, scene_instance_id, creator="", extra_info=None): - if not scene_instance_id: - raise ValueError("场景实例 ID 不能为空") - - instance = cls( - project_id=project_id, - template_id=template_id, - scene_instance_id=scene_instance_id, - creator=creator, - extra_info=extra_info, - ) - instance.save() - return instance diff --git a/gcloud/contrib/template_market/permission.py b/gcloud/contrib/template_market/permission.py index 434d503a4..b7720f852 100644 --- a/gcloud/contrib/template_market/permission.py +++ b/gcloud/contrib/template_market/permission.py @@ -12,24 +12,29 @@ """ import logging +from django.db.models import Q from rest_framework import permissions from gcloud.conf import settings from gcloud.contrib.template_market.models import TemplateSharedRecord +from gcloud.iam_auth import IAMMeta +from gcloud.iam_auth.utils import iam_multi_resource_auth_or_raise class TemplatePreviewPermission(permissions.BasePermission): def has_permission(self, request, view): - template_id = request.GET.get("template_id") - project_id = request.GET.get("project_id") - - if not template_id or not project_id: - logging.warning("Missing required parameters.") + try: + template_id = int(request.GET.get("template_id")) + project_id = int(request.GET.get("project_id")) + except (TypeError, ValueError): + logging.warning("Missing or invalid required parameters.") return False - record = TemplateSharedRecord.objects.filter(template_id=template_id, project_id=project_id).first() + record = TemplateSharedRecord.objects.filter( + Q(project_id=project_id) & Q(templates__contains=[template_id]) + ).first() if record is None: - logging.warning("template_id {} does not exist.".format(template_id)) + logging.warning("The specified template could not be found") return False return True @@ -37,4 +42,14 @@ def has_permission(self, request, view): class SharedProcessTemplatePermission(permissions.BasePermission): def has_permission(self, request, view): + username = request.user.username + serializer = view.serializer_class(data=request.data) + serializer.is_valid(raise_exception=True) + + template_id_list = [template.get("id") for template in serializer.validated_data["templates"]] + if view.action in ["create", "partial_update"]: + iam_multi_resource_auth_or_raise( + username, IAMMeta.FLOW_EDIT_ACTION, template_id_list, "resources_list_for_flows" + ) + return settings.ENABLE_TEMPLATE_MARKET diff --git a/gcloud/contrib/template_market/serializers.py b/gcloud/contrib/template_market/serializers.py index 9e5658815..baa887500 100644 --- a/gcloud/contrib/template_market/serializers.py +++ b/gcloud/contrib/template_market/serializers.py @@ -11,9 +11,11 @@ specific language governing permissions and limitations under the License. """ import json +import logging from rest_framework import serializers from gcloud.constants import DATETIME_FORMAT +from gcloud.contrib.template_market.models import TemplateSharedRecord class TemplatePreviewSerializer(serializers.Serializer): @@ -24,15 +26,71 @@ def get_pipeline_tree(self, obj): return json.dumps(obj.pipeline_tree) -class TemplateSharedRecordSerializer(serializers.Serializer): +class TemplateProjectBaseSerializer(serializers.Serializer): template_id = serializers.CharField(required=True, help_text="模板id") project_id = serializers.CharField(required=True, help_text="项目id") - name = serializers.CharField(required=True, help_text="场景名称") - code = serializers.CharField(required=True, help_text="场景标识") - category = serializers.CharField(required=True, help_text="场景分类") - risk_level = serializers.IntegerField(required=True, help_text="风险级别") - labels = serializers.ListField(child=serializers.IntegerField(), required=True, help_text="场景标签列表") - usage_content = serializers.CharField(required=True, help_text="使用说明") + + +class TemplateSharedRecordSerializer(serializers.ModelSerializer): + project_id = serializers.CharField(required=True, max_length=32, help_text="项目id") + templates = serializers.ListField(required=True, help_text="关联的模板列表") creator = serializers.CharField(required=False, max_length=32, help_text="创建者") create_at = serializers.DateTimeField(required=False, help_text="创建时间", format=DATETIME_FORMAT) + update_at = serializers.DateTimeField(required=False, help_text="更新时间", format=DATETIME_FORMAT) extra_info = serializers.JSONField(required=False, allow_null=True, help_text="额外信息") + id = serializers.IntegerField(required=False, help_text="共享实例id") + name = serializers.CharField(required=True, help_text="共享名称") + code = serializers.CharField(required=True, help_text="共享标识") + category = serializers.CharField(required=True, help_text="共享分类") + risk_level = serializers.IntegerField(required=True, help_text="风险级别") + usage_id = serializers.IntegerField(required=True, help_text="使用说明id") + labels = serializers.ListField(child=serializers.IntegerField(), required=True, help_text="共享标签列表") + usage_content = serializers.JSONField(required=True, help_text="使用说明") + + class Meta: + model = TemplateSharedRecord + fields = [ + "project_id", + "templates", + "creator", + "create_at", + "update_at", + "extra_info", + "labels", + "usage_content", + "id", + "name", + "code", + "category", + "risk_level", + "usage_id", + ] + + def convert_templates(self, templates): + return [template.get("id") for template in templates] + + def create(self, validated_data): + try: + validated_data["templates"] = self.convert_templates(validated_data["templates"]) + return TemplateSharedRecord.objects.create( + scene_shared_id=validated_data["id"], + project_id=validated_data["project_id"], + templates=validated_data["templates"], + creator=validated_data["creator"], + extra_info=validated_data["extra_info"], + ) + except Exception: + logging.exception("Failed to create model sharing record") + raise Exception("Failed to create model sharing record") + + def update(self, instance, validated_data): + try: + validated_data["templates"] = self.convert_templates(validated_data["templates"]) + instance.project_id = validated_data["project_id"] + instance.templates = validated_data["templates"] + instance.creator = validated_data["creator"] + instance.extra_info = validated_data["extra_info"] + instance.save() + except Exception: + logging.exception("Failed to update model sharing record") + raise Exception("Failed to update model sharing record") diff --git a/gcloud/contrib/template_market/utils.py b/gcloud/contrib/template_market/utils.py new file mode 100644 index 000000000..359b333fd --- /dev/null +++ b/gcloud/contrib/template_market/utils.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" + +import requests + +from gcloud.conf import settings + + +class MarketAPIClient: + def __init__(self): + self.base_url = settings.TEMPLATE_MARKET_API_URL + + def _get_url(self, endpoint): + return f"{self.base_url}{endpoint}" + + def get_template_list(self): + url = self._get_url("/sre_scene/flow_template_scene/") + response = requests.get(url) + return response.json() + + def create_template(self, data): + url = self._get_url("/sre_scene/flow_template_scene/") + response = requests.post(url, json=data) + return response.json() + + def patch_template(self, data, scene_shared_id): + url = self._get_url(f"/sre_scene/flow_template_scene/{scene_shared_id}/") + response = requests.patch(url, json=data) + return response.json() diff --git a/gcloud/contrib/template_market/viewsets.py b/gcloud/contrib/template_market/viewsets.py index b9aeb87fe..e35d2f3d0 100644 --- a/gcloud/contrib/template_market/viewsets.py +++ b/gcloud/contrib/template_market/viewsets.py @@ -11,19 +11,23 @@ specific language governing permissions and limitations under the License. """ import json -import requests import logging from rest_framework import viewsets from rest_framework.response import Response from rest_framework import permissions -from drf_yasg.utils import swagger_auto_schema -from gcloud.conf import settings from gcloud import err_code -from gcloud.contrib.template_market.serializers import TemplateSharedRecordSerializer, TemplatePreviewSerializer +from gcloud.conf import settings +from drf_yasg.utils import swagger_auto_schema +from gcloud.contrib.template_market.serializers import ( + TemplateSharedRecordSerializer, + TemplatePreviewSerializer, + TemplateProjectBaseSerializer, +) from gcloud.contrib.template_market.models import TemplateSharedRecord from gcloud.taskflow3.models import TaskTemplate +from gcloud.contrib.template_market.utils import MarketAPIClient from gcloud.contrib.template_market.permission import TemplatePreviewPermission, SharedProcessTemplatePermission @@ -33,7 +37,13 @@ class TemplatePreviewViewSet(viewsets.ViewSet): permission_classes = [permissions.IsAuthenticated, TemplatePreviewPermission] def retrieve(self, request, *args, **kwargs): - instance = self.queryset.get(id=request.GET.get("template_id"), project_id=request.GET.get("project_id")) + request_serializer = TemplateProjectBaseSerializer(data=request.GET) + request_serializer.is_valid(raise_exception=True) + + template_id = request_serializer.validated_data["template_id"] + project_id = request_serializer.validated_data["project_id"] + + instance = self.queryset.get(id=template_id, project_id=project_id) serializer = self.serializer_class(instance) return Response({"result": True, "data": serializer.data, "code": err_code.SUCCESS.code}) @@ -44,31 +54,46 @@ class SharedProcessTemplateViewSet(viewsets.ViewSet): serializer_class = TemplateSharedRecordSerializer permission_classes = [permissions.IsAuthenticated, SharedProcessTemplatePermission] - def _get_market_routing(self, market_url): - return f"{settings.TEMPLATE_MARKET_API_URL}/{market_url}" + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.market_client = MarketAPIClient() - def retrieve(self, request, *args, **kwargs): - project_id = request.GET.get("project_id") - template_id = request.GET.get("template_id") + def _build_template_data(self, serializer): - template_shared_obj = TemplateSharedRecord.objects.filter( - project_id=project_id, template_id=template_id - ).first() + data = { + "name": serializer.validated_data["name"], + "code": serializer.validated_data["code"], + "category": serializer.validated_data["category"], + "risk_level": serializer.validated_data["risk_level"], + "usage_id": serializer.validated_data["usage_id"], + "labels": serializer.validated_data["labels"], + "source_system": "bk_sops", + "project_code": serializer.validated_data["project_id"], + "templates": json.dumps(serializer.validated_data["templates"]), + "usage_content": serializer.validated_data["usage_content"], + } + scene_shared_id = serializer.validated_data.get("id") + if scene_shared_id: + data["id"] = scene_shared_id + return data + + def _get_processes_count(self, templates): + template_id_list = [template.get("id") for template in templates] + + template_objs = TaskTemplate.objects.filter(id__in=template_id_list) + + total_count = 0 + for template in template_objs: + activities = template.pipeline_tree.get("activities", []) + total_count += len(activities) + + return total_count + + def list(self, request, *args, **kwargs): + response_data = self.market_client.get_template_list() - if not template_shared_obj: - logging.exception(f"Template shared record not found, project_id: {project_id}, template_id: {template_id}") - return Response( - { - "result": False, - "message": "Template shared record not found", - "code": err_code.CONTENT_NOT_EXIST.code, - } - ) - url = self._get_market_routing(f"sre_scene/flow_template_scene/{template_shared_obj.scene_instance_id}/") - result = requests.get(url=url) - response_data = result.json() if not response_data["result"]: - logging.exception(f"Get template information from market failed, error code: {result.status_code}") + logging.exception(f"Get template information from market failed, error code: {response_data.get('code')}") return Response( {"result": False, "message": "Get template information failed", "code": err_code.OPERATION_FAIL.code} ) @@ -80,31 +105,20 @@ def create(self, request, *args, **kwargs): serializer = self.serializer_class(data=request.data) serializer.is_valid(raise_exception=True) - project_id = serializer.validated_data["project_id"] - template_id = serializer.validated_data["template_id"] - - task_template_obj = TaskTemplate.objects.filter(project_id=project_id, id=template_id).first() - if not task_template_obj: - logging.exception(f"Template with project_id {project_id} and template_id {template_id} not found.") - return Response({"result": False, "message": "Template not found", "code": err_code.CONTENT_NOT_EXIST.code}) - - url = self._get_market_routing("sre_scene/flow_template_scene/") + template_count = self._get_processes_count(serializer.validated_data["templates"]) + if template_count > settings.MAX_NUMBER_SHARED_PROCESSES: + return Response( + { + "result": False, + "message": "The number of selected templates exceeds the limit", + "code": err_code.OPERATION_FAIL.code, + } + ) - data = { - "name": serializer.validated_data["name"], - "code": serializer.validated_data["code"], - "category": serializer.validated_data["category"], - "risk_level": serializer.validated_data["risk_level"], - "labels": serializer.validated_data["labels"], - "source_system": "bk_sops", - "project_code": project_id, - "templates": json.dumps([{"id": template_id, "name": task_template_obj.name}]), - "usage_content": {"content": serializer.validated_data["usage_content"]}, - } + data = self._build_template_data(serializer) try: - result = requests.post(url, data=data) - response_data = result.json() - if not response_data["result"]: + response_data = self.market_client.create_template(data) + if not response_data.get("result"): return Response( { "result": False, @@ -112,17 +126,12 @@ def create(self, request, *args, **kwargs): "code": err_code.OPERATION_FAIL.code, } ) - TemplateSharedRecord.create( - project_id=project_id, - template_id=template_id, - scene_instance_id=response_data["data"]["id"], - creator=serializer.validated_data.get("creator"), - extra_info=serializer.validated_data.get("extra_info"), - ) + serializer.validated_data["id"] = response_data["data"]["id"] + serializer.create(serializer.validated_data) return Response( { "result": True, - "data": response_data, + "data": "response_data", "message": "Share template successfully", "code": err_code.SUCCESS.code, } @@ -130,3 +139,37 @@ def create(self, request, *args, **kwargs): except Exception as e: logging.exception("Share template failed: %s", e) return Response({"result": False, "message": "Share template failed", "code": err_code.OPERATION_FAIL.code}) + + @swagger_auto_schema(request_body=TemplateSharedRecordSerializer) + def partial_update(self, request, *args, **kwargs): + serializer = self.serializer_class(data=request.data, partial=True) + serializer.is_valid(raise_exception=True) + + scene_shared_id = serializer.validated_data.get("id") + + data = self._build_template_data(serializer) + try: + response_data = self.market_client.patch_template(data, scene_shared_id) + if not response_data.get("result"): + return Response( + { + "result": False, + "message": "Update template to sre store failed", + "code": err_code.OPERATION_FAIL.code, + } + ) + instance = self.queryset.get(scene_shared_id=scene_shared_id) + serializer.update(instance, serializer.validated_data) + return Response( + { + "result": True, + "data": response_data, + "message": "Share template successfully", + "code": err_code.SUCCESS.code, + } + ) + except Exception as e: + logging.exception("Failed to update scene template: %s", e) + return Response( + {"result": False, "message": "Failed to update scene template", "code": err_code.OPERATION_FAIL.code} + )