diff --git a/set_up.py b/set_up.py index 5ec8ad4..83bf541 100644 --- a/set_up.py +++ b/set_up.py @@ -11,7 +11,7 @@ def get_version(): - version = '1.0.0' + version = '1.0.1' return version diff --git a/tk/delta/adapter.py b/tk/delta/adapter.py index cc27fc7..ca6c2c3 100644 --- a/tk/delta/adapter.py +++ b/tk/delta/adapter.py @@ -3,7 +3,6 @@ # Copyright (c) Huawei Technologies Co., Ltd. 2022-2023, All rights reserved. from collections import OrderedDict -from tk.utils.version_utils import is_version_ge import mindspore as ms import mindspore.nn as nn @@ -13,14 +12,14 @@ from mindspore.ops import functional as F from tk.delta.delta_constants import VALID_TENSOR_DATATYPE -if is_version_ge(ms.__version__, '1.11.0'): +try: + from mindspore.nn.transformer.layers import _Linear, _args_type_validator_check, _valid_value_checks + from mindspore._checkparam import Validator +except: from mindformers.modules.layers import Linear, _args_type_validator_check, _valid_value_checks import mindspore._checkparam as Validator _Linear = Linear -else: - from mindspore.nn.transformer.layers import _Linear, _args_type_validator_check, _valid_value_checks - from mindspore._checkparam import Validator - + class AdapterLayer(nn.Cell): """ 定义微调算法adapter layer层,初始化adapter layer层参数,包括矩阵参数、激活层等。 diff --git a/tk/delta/lora.py b/tk/delta/lora.py index de772f8..957ddcd 100644 --- a/tk/delta/lora.py +++ b/tk/delta/lora.py @@ -3,8 +3,6 @@ # Copyright (c) Huawei Technologies Co., Ltd. 2022-2023, All rights reserved. import math -from tk.utils.version_utils import is_version_ge - import mindspore as ms import mindspore.nn as nn from mindspore import ops @@ -15,13 +13,14 @@ from mindspore.ops import functional as F from mindspore.common.initializer import initializer, HeUniform from tk.delta.delta_constants import VALID_TENSOR_DATATYPE +from tk.utils.version_control import get_dropout -if is_version_ge(ms.__version__, '1.11.0'): - import mindspore._checkparam as Validator - INC_LEFT = Validator.INC_LEFT -else: +try: from mindspore._checkparam import Validator, Rel INC_LEFT = Rel.INC_LEFT +except: + import mindspore._checkparam as Validator + INC_LEFT = Validator.INC_LEFT class LoRADense(nn.Dense): """Define a dense layer with LoRA structure. @@ -60,10 +59,7 @@ def __init__( # Define and initialize params self.lora_rank = lora_rank self.lora_alpha = lora_alpha - if is_version_ge(ms.__version__, '1.11.0'): - self.lora_dropout = nn.Dropout(p=lora_dropout) - else: - self.lora_dropout = nn.Dropout(keep_prob=1 - lora_dropout) + self.lora_dropout = get_dropout(lora_dropout) self.tk_delta_lora_a = Parameter( initializer(lora_a_init, [lora_rank, in_channels], param_init_type), name='tk_delta_lora_A') diff --git a/tk/delta/low_rank_adapter.py b/tk/delta/low_rank_adapter.py index f62d815..cf6a00f 100644 --- a/tk/delta/low_rank_adapter.py +++ b/tk/delta/low_rank_adapter.py @@ -3,7 +3,6 @@ # Copyright (c) Huawei Technologies Co., Ltd. 2022-2023, All rights reserved. import numbers -from tk.utils.version_utils import is_version_ge import mindspore as ms import mindspore.nn as nn @@ -17,12 +16,12 @@ from mindspore.ops import functional as F from tk.delta.delta_constants import VALID_TENSOR_DATATYPE -if is_version_ge(ms.__version__, '1.11.0'): - from mindformers.modules.layers import _args_type_validator_check, _valid_type_checks, _valid_value_checks - import mindspore._checkparam as Validator -else: +try: from mindspore.nn.transformer.layers import _args_type_validator_check, _valid_type_checks, _valid_value_checks from mindspore._checkparam import Validator +except: + from mindformers.modules.layers import _args_type_validator_check, _valid_type_checks, _valid_value_checks + import mindspore._checkparam as Validator class LowRankLinear(nn.Cell): diff --git a/tk/delta/prefix_layer.py b/tk/delta/prefix_layer.py index 865b1e0..6eb7ec2 100644 --- a/tk/delta/prefix_layer.py +++ b/tk/delta/prefix_layer.py @@ -5,14 +5,14 @@ import mindspore as ms import mindspore.nn as nn -from tk.utils.version_utils import is_version_ge +from tk.utils.version_control import get_dropout -if is_version_ge(ms.__version__, '1.11.0'): - import mindspore._checkparam as Validator - INC_LEFT = Validator.INC_LEFT -else: +try: from mindspore._checkparam import Validator, Rel INC_LEFT = Rel.INC_LEFT +except: + import mindspore._checkparam as Validator + INC_LEFT = Validator.INC_LEFT def check_multiple(param_dividend, value_dividend, param_divisor, value_divisor): @@ -55,10 +55,7 @@ def __init__(self, except ValueError as ex: raise ValueError(f"Invalid param [prefix_token_num] when initializing" f"PrefixLayer, error message:{str(ex)}") from ex - if is_version_ge(ms.__version__, '1.11.0'): - self.dropout = nn.Dropout(p=dropout_rate) - else: - self.dropout = nn.Dropout(keep_prob=1 - dropout_rate) + self.dropout = get_dropout(dropout_rate) self.past_value_reparam = None self.past_key_reparam = None self.__define_network() diff --git a/tk/utils/version_control.py b/tk/utils/version_control.py new file mode 100644 index 0000000..5408248 --- /dev/null +++ b/tk/utils/version_control.py @@ -0,0 +1,10 @@ +import mindspore as ms +from mindspore import nn +from .version_utils import is_version_ge + +def get_dropout(dropout_prob): + if is_version_ge(ms.__version__, '1.11.0'): + dropout = nn.Dropout(p=dropout_prob) + else: + dropout = nn.Dropout(keep_prob=1 - dropout_prob) + return dropout \ No newline at end of file