feat: local data source import

src/bk-user/bkuser/apis/web/data_source/mixins.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available.
+Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from bkuser.apis.web.mixins import CurrentUserTenantMixin
+from bkuser.apps.data_source.models import DataSource
+
+
+class CurrentUserTenantDataSourceMixin(CurrentUserTenantMixin):
+    """获取当前用户所在租户下属数据源"""
+
+    lookup_url_kwarg = "id"
+
+    def get_queryset(self):
+        return DataSource.objects.filter(owner_tenant_id=self.get_current_tenant_id())

src/bk-user/bkuser/apis/web/data_source/serializers.py

@@ -193,4 +193,12 @@ class LocalDataSourceImportInputSLZ(serializers.Serializer):
     """本地数据源导入"""
 
     file = serializers.FileField(help_text="数据源用户信息文件（Excel 格式）")
-    overwrite = serializers.BooleanField(help_text="允许对同名用户覆盖更新")
+    overwrite = serializers.BooleanField(help_text="允许对同名用户覆盖更新", default=False)
+
+
+class LocalDataSourceImportOutputSLZ(serializers.Serializer):
+    """本地数据源导入结果"""
+
+    task_id = serializers.CharField(help_text="任务 ID")
+    status = serializers.CharField(help_text="任务状态")
+    summary = serializers.CharField(help_text="任务执行结果概述")

src/bk-user/bkuser/apis/web/data_source/views.py

@@ -8,13 +8,18 @@
 an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 specific language governing permissions and limitations under the License.
 """
+import logging
+
+import openpyxl
 from django.conf import settings
 from django.db import transaction
 from django.utils.translation import gettext_lazy as _
 from drf_yasg.utils import swagger_auto_schema
+from openpyxl.utils.exceptions import InvalidFileException
 from rest_framework import generics, status
 from rest_framework.response import Response
 
+from bkuser.apis.web.data_source.mixins import CurrentUserTenantDataSourceMixin
 from bkuser.apis.web.data_source.serializers import (
     DataSourceCreateInputSLZ,
     DataSourceCreateOutputSLZ,
@@ -26,18 +31,25 @@
     DataSourceSwitchStatusOutputSLZ,
     DataSourceTestConnectionOutputSLZ,
     DataSourceUpdateInputSLZ,
+    LocalDataSourceImportInputSLZ,
+    LocalDataSourceImportOutputSLZ,
 )
 from bkuser.apis.web.mixins import CurrentUserTenantMixin
 from bkuser.apps.data_source.constants import DataSourceStatus
 from bkuser.apps.data_source.exporter import DataSourceUserExporter
 from bkuser.apps.data_source.models import DataSource, DataSourcePlugin
 from bkuser.apps.data_source.plugins.constants import DATA_SOURCE_PLUGIN_CONFIG_SCHEMA_MAP
 from bkuser.apps.data_source.signals import post_create_data_source, post_update_data_source
+from bkuser.apps.sync.constants import SyncTaskTrigger
+from bkuser.apps.sync.data_models import DataSourceSyncOptions
+from bkuser.apps.sync.manager import DataSourceSyncManager
 from bkuser.biz.data_source_plugin import DefaultPluginConfigProvider
 from bkuser.common.error_codes import error_codes
 from bkuser.common.response import convert_workbook_to_response
 from bkuser.common.views import ExcludePatchAPIViewMixin, ExcludePutAPIViewMixin
 
+logger = logging.getLogger(__name__)
+
 
 class DataSourcePluginListApi(generics.ListAPIView):
     queryset = DataSourcePlugin.objects.all()
@@ -131,13 +143,11 @@ def post(self, request, *args, **kwargs):
         )
 
 
-class DataSourceRetrieveUpdateApi(CurrentUserTenantMixin, ExcludePatchAPIViewMixin, generics.RetrieveUpdateAPIView):
+class DataSourceRetrieveUpdateApi(
+    CurrentUserTenantDataSourceMixin, ExcludePatchAPIViewMixin, generics.RetrieveUpdateAPIView
+):
     pagination_class = None
     serializer_class = DataSourceRetrieveOutputSLZ
-    lookup_url_kwarg = "id"
-
-    def get_queryset(self):
-        return DataSource.objects.filter(owner_tenant_id=self.get_current_tenant_id())
 
     @swagger_auto_schema(
         tags=["data_source"],
@@ -179,14 +189,10 @@ def put(self, request, *args, **kwargs):
         return Response(status=status.HTTP_204_NO_CONTENT)
 
 
-class DataSourceTestConnectionApi(CurrentUserTenantMixin, generics.RetrieveAPIView):
+class DataSourceTestConnectionApi(CurrentUserTenantDataSourceMixin, generics.RetrieveAPIView):
     """数据源连通性测试"""
 
     serializer_class = DataSourceTestConnectionOutputSLZ
-    lookup_url_kwarg = "id"
-
-    def get_queryset(self):
-        return DataSource.objects.filter(owner_tenant_id=self.get_current_tenant_id())
 
     @swagger_auto_schema(
         tags=["data_source"],
@@ -219,14 +225,10 @@ def get(self, request, *args, **kwargs):
         return Response(DataSourceTestConnectionOutputSLZ(instance=mock_data).data)
 
 
-class DataSourceSwitchStatusApi(CurrentUserTenantMixin, ExcludePutAPIViewMixin, generics.UpdateAPIView):
+class DataSourceSwitchStatusApi(CurrentUserTenantDataSourceMixin, ExcludePutAPIViewMixin, generics.UpdateAPIView):
     """切换数据源状态（启/停）"""
 
     serializer_class = DataSourceSwitchStatusOutputSLZ
-    lookup_url_kwarg = "id"
-
-    def get_queryset(self):
-        return DataSource.objects.filter(owner_tenant_id=self.get_current_tenant_id())
 
     @swagger_auto_schema(
         tags=["data_source"],
@@ -246,14 +248,9 @@ def patch(self, request, *args, **kwargs):
         return Response(DataSourceSwitchStatusOutputSLZ(instance={"status": data_source.status.value}).data)
 
 
-class DataSourceTemplateApi(CurrentUserTenantMixin, generics.ListAPIView):
+class DataSourceTemplateApi(CurrentUserTenantDataSourceMixin, generics.ListAPIView):
     """获取本地数据源数据导入模板"""
 
-    lookup_url_kwarg = "id"
-
-    def get_queryset(self):
-        return DataSource.objects.filter(owner_tenant_id=self.get_current_tenant_id())
-
     @swagger_auto_schema(
         tags=["data_source"],
         operation_description="下载数据源导入模板",
@@ -270,14 +267,9 @@ def get(self, request, *args, **kwargs):
         return convert_workbook_to_response(workbook, f"{settings.EXPORT_EXCEL_FILENAME_PREFIX}_org_tmpl.xlsx")
 
 
-class DataSourceExportApi(CurrentUserTenantMixin, generics.ListAPIView):
+class DataSourceExportApi(CurrentUserTenantDataSourceMixin, generics.ListAPIView):
     """本地数据源用户导出"""
 
-    lookup_url_kwarg = "id"
-
-    def get_queryset(self):
-        return DataSource.objects.filter(owner_tenant_id=self.get_current_tenant_id())
-
     @swagger_auto_schema(
         tags=["data_source"],
         operation_description="下载本地数据源用户数据",
@@ -293,28 +285,45 @@ def get(self, request, *args, **kwargs):
         return convert_workbook_to_response(workbook, f"{settings.EXPORT_EXCEL_FILENAME_PREFIX}_org_data.xlsx")
 
 
-class DataSourceImportApi(CurrentUserTenantMixin, generics.CreateAPIView):
+class DataSourceImportApi(CurrentUserTenantDataSourceMixin, generics.CreateAPIView):
     """从 Excel 导入数据源用户数据"""
 
-    lookup_url_kwarg = "id"
-
-    def get_queryset(self):
-        return DataSource.objects.filter(owner_tenant_id=self.get_current_tenant_id())
-
     @swagger_auto_schema(
         tags=["data_source"],
         operation_description="本地数据源用户数据导入",
-        responses={status.HTTP_200_OK: ""},
+        request_body=LocalDataSourceImportInputSLZ(),
+        responses={status.HTTP_200_OK: LocalDataSourceImportOutputSLZ()},
     )
     def post(self, request, *args, **kwargs):
         """从 Excel 导入数据源用户数据"""
+        slz = LocalDataSourceImportInputSLZ(data=request.data)
+        slz.is_valid(raise_exception=True)
+        data = slz.validated_data
+
         data_source = self.get_object()
         if not data_source.is_local:
             raise error_codes.DATA_SOURCE_OPERATION_UNSUPPORTED.f(_("仅本地数据源支持导入功能"))
 
-        # TODO (su) 调用本地数据源插件，对 workbook 进行解析，构造出数据源用户数据
+        # Request file 转换成 openpyxl.workbook
+        try:
+            workbook = openpyxl.load_workbook(data["file"])
+        except InvalidFileException:
+            logger.exception("本地数据源导入失败")
+            raise error_codes.DATA_SOURCE_IMPORT_FAILED.f(_("文件格式异常"))
+
+        options = DataSourceSyncOptions(
+            operator=request.user.username,
+            overwrite=data["overwrite"],
+            async_run=False,
+            trigger=SyncTaskTrigger.MANUAL,
+        )
 
-        return Response()
+        task = DataSourceSyncManager(data_source, options).execute(context={"workbook": workbook})
+        return Response(
+            LocalDataSourceImportOutputSLZ(
+                instance={"task_id": task.id, "status": task.status, "summary": task.summary}
+            ).data
+        )
 
 
 class DataSourceSyncApi(generics.CreateAPIView):

src/bk-user/bkuser/apps/data_source/migrations/0004_alter_localdatasourceidentityinfo_password.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available.
+Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+# Generated by Django 3.2.20 on 2023-09-12 03:18
+
+import blue_krill.models.fields
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data_source', '0003_auto_20230831_1552'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='localdatasourceidentityinfo',
+            name='password',
+            field=blue_krill.models.fields.EncryptField(blank=True, default='', max_length=255, null=True),
+        ),
+    ]

src/bk-user/bkuser/apps/data_source/models.py

@@ -10,6 +10,7 @@
 """
 import uuid
 
+from blue_krill.models.fields import EncryptField
 from django.conf import settings
 from django.db import models
 from mptt.models import MPTTModel, TreeForeignKey
@@ -90,7 +91,7 @@ class LocalDataSourceIdentityInfo(TimestampedModel):
     """
 
     user = models.OneToOneField(DataSourceUser, on_delete=models.CASCADE)
-    password = models.CharField("用户密码", null=True, blank=True, default="", max_length=255)
+    password = EncryptField("用户密码", null=True, blank=True, default="", max_length=255)
     password_updated_at = models.DateTimeField("密码最后更新时间", null=True, blank=True)
     password_expired_at = models.DateTimeField("密码过期时间", null=True, blank=True)
 

src/bk-user/bkuser/apps/data_source/plugins/base.py

@@ -18,13 +18,13 @@ class BaseDataSourcePlugin(ABC):
     """数据源插件基类"""
 
     @abstractmethod
-    def fetch_departments(self) -> List[RawDataSourceDepartment]:
-        """获取部门信息"""
+    def fetch_users(self) -> List[RawDataSourceUser]:
+        """获取用户信息"""
         ...
 
     @abstractmethod
-    def fetch_users(self) -> List[RawDataSourceUser]:
-        """获取用户信息"""
+    def fetch_departments(self) -> List[RawDataSourceDepartment]:
+        """获取部门信息"""
         ...
 
     @abstractmethod

src/bk-user/bkuser/apps/data_source/plugins/constants.py

@@ -11,8 +11,14 @@
 
 from bkuser.apps.data_source.constants import DataSourcePluginEnum
 from bkuser.apps.data_source.plugins.local.models import LocalDataSourcePluginConfig
+from bkuser.apps.data_source.plugins.local.plugin import LocalDataSourcePlugin
 from bkuser.utils.pydantic import gen_openapi_schema
 
+# 数据源插件类映射表
+DATA_SOURCE_PLUGIN_CLASS_MAP = {
+    DataSourcePluginEnum.LOCAL: LocalDataSourcePlugin,
+}
+
 # 数据源插件配置类映射表
 DATA_SOURCE_PLUGIN_CONFIG_CLASS_MAP = {
     DataSourcePluginEnum.LOCAL: LocalDataSourcePluginConfig,

src/bk-user/bkuser/apps/sync/pollers.py → src/bk-user/bkuser/apps/data_source/plugins/exceptions.py

@@ -9,16 +9,6 @@
 specific language governing permissions and limitations under the License.
 """
 
-# TODO (su) Poller 会基于 blue-krill 的能力，提供各类同步任务
 
-
-class DataSourceSyncTaskPoller:
-    """数据源同步任务上下文管理器"""
-
-    ...
-
-
-class TenantSyncTaskPoller:
-    """租户同步任务上下文管理器"""
-
-    ...
+class BaseDataSourcePluginError(Exception):
+    """数据源插件基础异常"""

src/bk-user/bkuser/apps/data_source/plugins/local/exceptions.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+"""
+TencentBlueKing is pleased to support the open source community by making 蓝鲸智云-用户管理(Bk-User) available.
+Copyright (C) 2017-2021 THL A29 Limited, a Tencent company. All rights reserved.
+Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at http://opensource.org/licenses/MIT
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+"""
+from bkuser.apps.data_source.plugins.exceptions import BaseDataSourcePluginError
+
+
+class LocalDataSourcePluginError(BaseDataSourcePluginError):
+    """本地数据源插件基础异常"""
+
+
+class UserSheetNotExists(LocalDataSourcePluginError):
+    """待导入文件中不存在用户表"""
+
+
+class SheetColumnsNotMatch(LocalDataSourcePluginError):
+    """待导入文件中用户表列不匹配"""
+
+
+class CustomColumnNameInvalid(LocalDataSourcePluginError):
+    """待导入文件中动态字段列名不合法"""
+
+
+class DuplicateColumnName(LocalDataSourcePluginError):
+    """待导入文件中存在重复列名"""
+
+
+class DuplicateUsername(LocalDataSourcePluginError):
+    """待导入文件中存在重复用户"""