From 4be031b91a533f98a5347c5ae64304371d11ad99 Mon Sep 17 00:00:00 2001 From: Jingshu918 <138486531+Jingshu918@users.noreply.github.com> Date: Thu, 19 Sep 2024 14:39:19 +0800 Subject: [PATCH] [DataFactory]Support New Features In ADF (#26070) --- .../Generated/Models/IcebergDataset.cs | 93 +++++++++++++++++++ .../Models/IcebergDatasetTypeProperties.cs | 64 +++++++++++++ .../Generated/Models/IcebergSink.cs | 83 +++++++++++++++++ .../Generated/Models/IcebergWriteSettings.cs | 43 +++++++++ .../Generated/Models/MariaDBLinkedService.cs | 42 ++++++++- .../MariaDBLinkedServiceTypeProperties.cs | 42 ++++++++- .../DataFactory.Management.Sdk/README.md | 2 +- src/DataFactory/DataFactory/ChangeLog.md | 2 + 8 files changed, 364 insertions(+), 7 deletions(-) create mode 100644 src/DataFactory/DataFactory.Management.Sdk/Generated/Models/IcebergDataset.cs create mode 100644 src/DataFactory/DataFactory.Management.Sdk/Generated/Models/IcebergDatasetTypeProperties.cs create mode 100644 src/DataFactory/DataFactory.Management.Sdk/Generated/Models/IcebergSink.cs create mode 100644 src/DataFactory/DataFactory.Management.Sdk/Generated/Models/IcebergWriteSettings.cs diff --git a/src/DataFactory/DataFactory.Management.Sdk/Generated/Models/IcebergDataset.cs b/src/DataFactory/DataFactory.Management.Sdk/Generated/Models/IcebergDataset.cs new file mode 100644 index 000000000000..60ee1a245e4e --- /dev/null +++ b/src/DataFactory/DataFactory.Management.Sdk/Generated/Models/IcebergDataset.cs @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. +// Code generated by Microsoft (R) AutoRest Code Generator. +// Changes may cause incorrect behavior and will be lost if the code is regenerated. + +namespace Microsoft.Azure.Management.DataFactory.Models +{ + using System.Linq; + + /// + /// Iceberg dataset. + /// + [Newtonsoft.Json.JsonObject("Iceberg")] + [Microsoft.Rest.Serialization.JsonTransformation] + public partial class IcebergDataset : Dataset + { + /// + /// Initializes a new instance of the IcebergDataset class. + /// + public IcebergDataset() + { + this.LinkedServiceName = new LinkedServiceReference(); + CustomInit(); + } + + /// + /// Initializes a new instance of the IcebergDataset class. + /// + + /// The Azure Data Factory nested object which identifies data within different + /// data stores, such as tables, files, folders, and documents. + /// + + /// Dataset description. + /// + + /// Columns that define the structure of the dataset. Type: array (or + /// Expression with resultType array), itemType: DatasetDataElement. + /// + + /// Columns that define the physical type schema of the dataset. Type: array + /// (or Expression with resultType array), itemType: DatasetSchemaDataElement. + /// + + /// Linked service reference. + /// + + /// Parameters for dataset. + /// + + /// List of tags that can be used for describing the Dataset. + /// + + /// The folder that this Dataset is in. If not specified, Dataset will appear + /// at the root level. + /// + + /// The location of the iceberg storage. Setting a file name is not allowed for + /// iceberg format. + /// + public IcebergDataset(LinkedServiceReference linkedServiceName, System.Collections.Generic.IDictionary additionalProperties = default(System.Collections.Generic.IDictionary), string description = default(string), object structure = default(object), object schema = default(object), System.Collections.Generic.IDictionary parameters = default(System.Collections.Generic.IDictionary), System.Collections.Generic.IList annotations = default(System.Collections.Generic.IList), DatasetFolder folder = default(DatasetFolder), DatasetLocation location = default(DatasetLocation)) + + : base(linkedServiceName, additionalProperties, description, structure, schema, parameters, annotations, folder) + { + this.Location = location; + CustomInit(); + } + + /// + /// An initialization method that performs custom operations like setting defaults + /// + partial void CustomInit(); + + + /// + /// Gets or sets the location of the iceberg storage. Setting a file name is + /// not allowed for iceberg format. + /// + [Newtonsoft.Json.JsonProperty(PropertyName = "typeProperties.location")] + public DatasetLocation Location {get; set; } + /// + /// Validate the object. + /// + /// + /// Thrown if validation fails + /// + public override void Validate() + { + base.Validate(); + + } + } +} \ No newline at end of file diff --git a/src/DataFactory/DataFactory.Management.Sdk/Generated/Models/IcebergDatasetTypeProperties.cs b/src/DataFactory/DataFactory.Management.Sdk/Generated/Models/IcebergDatasetTypeProperties.cs new file mode 100644 index 000000000000..4cf60aa6a79b --- /dev/null +++ b/src/DataFactory/DataFactory.Management.Sdk/Generated/Models/IcebergDatasetTypeProperties.cs @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. +// Code generated by Microsoft (R) AutoRest Code Generator. +// Changes may cause incorrect behavior and will be lost if the code is regenerated. + +namespace Microsoft.Azure.Management.DataFactory.Models +{ + using System.Linq; + + /// + /// Iceberg dataset properties. + /// + public partial class IcebergDatasetTypeProperties + { + /// + /// Initializes a new instance of the IcebergDatasetTypeProperties class. + /// + public IcebergDatasetTypeProperties() + { + CustomInit(); + } + + /// + /// Initializes a new instance of the IcebergDatasetTypeProperties class. + /// + + /// The location of the iceberg storage. Setting a file name is not allowed for + /// iceberg format. + /// + public IcebergDatasetTypeProperties(DatasetLocation location) + + { + this.Location = location; + CustomInit(); + } + + /// + /// An initialization method that performs custom operations like setting defaults + /// + partial void CustomInit(); + + + /// + /// Gets or sets the location of the iceberg storage. Setting a file name is + /// not allowed for iceberg format. + /// + [Newtonsoft.Json.JsonProperty(PropertyName = "location")] + public DatasetLocation Location {get; set; } + /// + /// Validate the object. + /// + /// + /// Thrown if validation fails + /// + public virtual void Validate() + { + if (this.Location == null) + { + throw new Microsoft.Rest.ValidationException(Microsoft.Rest.ValidationRules.CannotBeNull, "Location"); + } + + } + } +} \ No newline at end of file diff --git a/src/DataFactory/DataFactory.Management.Sdk/Generated/Models/IcebergSink.cs b/src/DataFactory/DataFactory.Management.Sdk/Generated/Models/IcebergSink.cs new file mode 100644 index 000000000000..e8924ddcf811 --- /dev/null +++ b/src/DataFactory/DataFactory.Management.Sdk/Generated/Models/IcebergSink.cs @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. +// Code generated by Microsoft (R) AutoRest Code Generator. +// Changes may cause incorrect behavior and will be lost if the code is regenerated. + +namespace Microsoft.Azure.Management.DataFactory.Models +{ + using System.Linq; + + /// + /// A copy activity Iceberg sink. + /// + [Newtonsoft.Json.JsonObject("IcebergSink")] + public partial class IcebergSink : CopySink + { + /// + /// Initializes a new instance of the IcebergSink class. + /// + public IcebergSink() + { + CustomInit(); + } + + /// + /// Initializes a new instance of the IcebergSink class. + /// + + /// Write batch size. Type: integer (or Expression with resultType integer), + /// minimum: 0. + /// + + /// Write batch timeout. Type: string (or Expression with resultType string), + /// pattern: ((\d+)\.)?(\d\d):(60|([0-5][0-9])):(60|([0-5][0-9])). + /// + + /// Sink retry count. Type: integer (or Expression with resultType integer). + /// + + /// Sink retry wait. Type: string (or Expression with resultType string), + /// pattern: ((\d+)\.)?(\d\d):(60|([0-5][0-9])):(60|([0-5][0-9])). + /// + + /// The maximum concurrent connection count for the sink data store. Type: + /// integer (or Expression with resultType integer). + /// + + /// If true, disable data store metrics collection. Default is false. Type: + /// boolean (or Expression with resultType boolean). + /// + + /// Iceberg store settings. + /// + + /// Iceberg format settings. + /// + public IcebergSink(object writeBatchSize = default(object), object writeBatchTimeout = default(object), object sinkRetryCount = default(object), object sinkRetryWait = default(object), object maxConcurrentConnections = default(object), object disableMetricsCollection = default(object), StoreWriteSettings storeSettings = default(StoreWriteSettings), IcebergWriteSettings formatSettings = default(IcebergWriteSettings)) + + : base(writeBatchSize, writeBatchTimeout, sinkRetryCount, sinkRetryWait, maxConcurrentConnections, disableMetricsCollection) + { + this.StoreSettings = storeSettings; + this.FormatSettings = formatSettings; + CustomInit(); + } + + /// + /// An initialization method that performs custom operations like setting defaults + /// + partial void CustomInit(); + + + /// + /// Gets or sets iceberg store settings. + /// + [Newtonsoft.Json.JsonProperty(PropertyName = "storeSettings")] + public StoreWriteSettings StoreSettings {get; set; } + + /// + /// Gets or sets iceberg format settings. + /// + [Newtonsoft.Json.JsonProperty(PropertyName = "formatSettings")] + public IcebergWriteSettings FormatSettings {get; set; } + } +} \ No newline at end of file diff --git a/src/DataFactory/DataFactory.Management.Sdk/Generated/Models/IcebergWriteSettings.cs b/src/DataFactory/DataFactory.Management.Sdk/Generated/Models/IcebergWriteSettings.cs new file mode 100644 index 000000000000..b6eeb0e75013 --- /dev/null +++ b/src/DataFactory/DataFactory.Management.Sdk/Generated/Models/IcebergWriteSettings.cs @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. +// Code generated by Microsoft (R) AutoRest Code Generator. +// Changes may cause incorrect behavior and will be lost if the code is regenerated. + +namespace Microsoft.Azure.Management.DataFactory.Models +{ + using System.Linq; + + /// + /// Iceberg write settings. + /// + [Newtonsoft.Json.JsonObject("IcebergWriteSettings")] + public partial class IcebergWriteSettings : FormatWriteSettings + { + /// + /// Initializes a new instance of the IcebergWriteSettings class. + /// + public IcebergWriteSettings() + { + CustomInit(); + } + + /// + /// Initializes a new instance of the IcebergWriteSettings class. + /// + + /// Format write settings. + /// + public IcebergWriteSettings(System.Collections.Generic.IDictionary additionalProperties = default(System.Collections.Generic.IDictionary)) + + : base(additionalProperties) + { + CustomInit(); + } + + /// + /// An initialization method that performs custom operations like setting defaults + /// + partial void CustomInit(); + + } +} \ No newline at end of file diff --git a/src/DataFactory/DataFactory.Management.Sdk/Generated/Models/MariaDBLinkedService.cs b/src/DataFactory/DataFactory.Management.Sdk/Generated/Models/MariaDBLinkedService.cs index 38a19f8a4a72..d55e79baaa65 100644 --- a/src/DataFactory/DataFactory.Management.Sdk/Generated/Models/MariaDBLinkedService.cs +++ b/src/DataFactory/DataFactory.Management.Sdk/Generated/Models/MariaDBLinkedService.cs @@ -47,7 +47,8 @@ public MariaDBLinkedService() /// The version of the MariaDB driver. Type: string. V1 or empty for legacy /// driver, V2 for new driver. V1 can support connection string and property - /// bag, V2 can only support connection string. + /// bag, V2 can only support connection string. The legacy driver is scheduled + /// for deprecation by October 2024. /// /// An ODBC connection string. Type: string, SecureString or @@ -66,13 +67,25 @@ public MariaDBLinkedService() /// Database name for connection. Type: string. /// + /// This option specifies whether the driver uses TLS encryption and + /// verification when connecting to MariaDB. E.g., SSLMode=<0/1/2/3/4>. + /// Options: DISABLED (0) / PREFERRED (1) (Default) / REQUIRED (2) / VERIFY_CA + /// (3) / VERIFY_IDENTITY (4), REQUIRED (2) is recommended to only allow + /// connections encrypted with SSL/TLS. + /// + + /// This option specifies whether to use a CA certificate from the system trust + /// store, or from a specified PEM file. E.g. UseSystemTrustStore=<0/1>; + /// Options: Enabled (1) / Disabled (0) (Default) + /// + /// The Azure key vault secret reference of password in connection string. /// /// The encrypted credential used for authentication. Credentials are encrypted /// using the integration runtime credential manager. Type: string. /// - public MariaDBLinkedService(System.Collections.Generic.IDictionary additionalProperties = default(System.Collections.Generic.IDictionary), string version = default(string), IntegrationRuntimeReference connectVia = default(IntegrationRuntimeReference), string description = default(string), System.Collections.Generic.IDictionary parameters = default(System.Collections.Generic.IDictionary), System.Collections.Generic.IList annotations = default(System.Collections.Generic.IList), object driverVersion = default(object), object connectionString = default(object), object server = default(object), object port = default(object), object username = default(object), object database = default(object), AzureKeyVaultSecretReference password = default(AzureKeyVaultSecretReference), string encryptedCredential = default(string)) + public MariaDBLinkedService(System.Collections.Generic.IDictionary additionalProperties = default(System.Collections.Generic.IDictionary), string version = default(string), IntegrationRuntimeReference connectVia = default(IntegrationRuntimeReference), string description = default(string), System.Collections.Generic.IDictionary parameters = default(System.Collections.Generic.IDictionary), System.Collections.Generic.IList annotations = default(System.Collections.Generic.IList), object driverVersion = default(object), object connectionString = default(object), object server = default(object), object port = default(object), object username = default(object), object database = default(object), object sslMode = default(object), object useSystemTrustStore = default(object), AzureKeyVaultSecretReference password = default(AzureKeyVaultSecretReference), string encryptedCredential = default(string)) : base(additionalProperties, version, connectVia, description, parameters, annotations) { @@ -82,6 +95,8 @@ public MariaDBLinkedService() this.Port = port; this.Username = username; this.Database = database; + this.SslMode = sslMode; + this.UseSystemTrustStore = useSystemTrustStore; this.Password = password; this.EncryptedCredential = encryptedCredential; CustomInit(); @@ -96,7 +111,8 @@ public MariaDBLinkedService() /// /// Gets or sets the version of the MariaDB driver. Type: string. V1 or empty /// for legacy driver, V2 for new driver. V1 can support connection string and - /// property bag, V2 can only support connection string. + /// property bag, V2 can only support connection string. The legacy driver is + /// scheduled for deprecation by October 2024. /// [Newtonsoft.Json.JsonProperty(PropertyName = "typeProperties.driverVersion")] public object DriverVersion {get; set; } @@ -132,6 +148,24 @@ public MariaDBLinkedService() [Newtonsoft.Json.JsonProperty(PropertyName = "typeProperties.database")] public object Database {get; set; } + /// + /// Gets or sets this option specifies whether the driver uses TLS encryption + /// and verification when connecting to MariaDB. E.g., SSLMode=<0/1/2/3/4>. + /// Options: DISABLED (0) / PREFERRED (1) (Default) / REQUIRED (2) / VERIFY_CA + /// (3) / VERIFY_IDENTITY (4), REQUIRED (2) is recommended to only allow + /// connections encrypted with SSL/TLS. + /// + [Newtonsoft.Json.JsonProperty(PropertyName = "typeProperties.sslMode")] + public object SslMode {get; set; } + + /// + /// Gets or sets this option specifies whether to use a CA certificate from the + /// system trust store, or from a specified PEM file. E.g. + /// UseSystemTrustStore=<0/1>; Options: Enabled (1) / Disabled (0) (Default) + /// + [Newtonsoft.Json.JsonProperty(PropertyName = "typeProperties.useSystemTrustStore")] + public object UseSystemTrustStore {get; set; } + /// /// Gets or sets the Azure key vault secret reference of password in connection /// string. @@ -161,6 +195,8 @@ public override void Validate() + + if (this.Password != null) { this.Password.Validate(); diff --git a/src/DataFactory/DataFactory.Management.Sdk/Generated/Models/MariaDBLinkedServiceTypeProperties.cs b/src/DataFactory/DataFactory.Management.Sdk/Generated/Models/MariaDBLinkedServiceTypeProperties.cs index 7166e8beb25e..674521a2a4fc 100644 --- a/src/DataFactory/DataFactory.Management.Sdk/Generated/Models/MariaDBLinkedServiceTypeProperties.cs +++ b/src/DataFactory/DataFactory.Management.Sdk/Generated/Models/MariaDBLinkedServiceTypeProperties.cs @@ -26,7 +26,8 @@ public MariaDBLinkedServiceTypeProperties() /// The version of the MariaDB driver. Type: string. V1 or empty for legacy /// driver, V2 for new driver. V1 can support connection string and property - /// bag, V2 can only support connection string. + /// bag, V2 can only support connection string. The legacy driver is scheduled + /// for deprecation by October 2024. /// /// An ODBC connection string. Type: string, SecureString or @@ -45,13 +46,25 @@ public MariaDBLinkedServiceTypeProperties() /// Database name for connection. Type: string. /// + /// This option specifies whether the driver uses TLS encryption and + /// verification when connecting to MariaDB. E.g., SSLMode=<0/1/2/3/4>. + /// Options: DISABLED (0) / PREFERRED (1) (Default) / REQUIRED (2) / VERIFY_CA + /// (3) / VERIFY_IDENTITY (4), REQUIRED (2) is recommended to only allow + /// connections encrypted with SSL/TLS. + /// + + /// This option specifies whether to use a CA certificate from the system trust + /// store, or from a specified PEM file. E.g. UseSystemTrustStore=<0/1>; + /// Options: Enabled (1) / Disabled (0) (Default) + /// + /// The Azure key vault secret reference of password in connection string. /// /// The encrypted credential used for authentication. Credentials are encrypted /// using the integration runtime credential manager. Type: string. /// - public MariaDBLinkedServiceTypeProperties(object driverVersion = default(object), object connectionString = default(object), object server = default(object), object port = default(object), object username = default(object), object database = default(object), AzureKeyVaultSecretReference password = default(AzureKeyVaultSecretReference), string encryptedCredential = default(string)) + public MariaDBLinkedServiceTypeProperties(object driverVersion = default(object), object connectionString = default(object), object server = default(object), object port = default(object), object username = default(object), object database = default(object), object sslMode = default(object), object useSystemTrustStore = default(object), AzureKeyVaultSecretReference password = default(AzureKeyVaultSecretReference), string encryptedCredential = default(string)) { this.DriverVersion = driverVersion; @@ -60,6 +73,8 @@ public MariaDBLinkedServiceTypeProperties() this.Port = port; this.Username = username; this.Database = database; + this.SslMode = sslMode; + this.UseSystemTrustStore = useSystemTrustStore; this.Password = password; this.EncryptedCredential = encryptedCredential; CustomInit(); @@ -74,7 +89,8 @@ public MariaDBLinkedServiceTypeProperties() /// /// Gets or sets the version of the MariaDB driver. Type: string. V1 or empty /// for legacy driver, V2 for new driver. V1 can support connection string and - /// property bag, V2 can only support connection string. + /// property bag, V2 can only support connection string. The legacy driver is + /// scheduled for deprecation by October 2024. /// [Newtonsoft.Json.JsonProperty(PropertyName = "driverVersion")] public object DriverVersion {get; set; } @@ -110,6 +126,24 @@ public MariaDBLinkedServiceTypeProperties() [Newtonsoft.Json.JsonProperty(PropertyName = "database")] public object Database {get; set; } + /// + /// Gets or sets this option specifies whether the driver uses TLS encryption + /// and verification when connecting to MariaDB. E.g., SSLMode=<0/1/2/3/4>. + /// Options: DISABLED (0) / PREFERRED (1) (Default) / REQUIRED (2) / VERIFY_CA + /// (3) / VERIFY_IDENTITY (4), REQUIRED (2) is recommended to only allow + /// connections encrypted with SSL/TLS. + /// + [Newtonsoft.Json.JsonProperty(PropertyName = "sslMode")] + public object SslMode {get; set; } + + /// + /// Gets or sets this option specifies whether to use a CA certificate from the + /// system trust store, or from a specified PEM file. E.g. + /// UseSystemTrustStore=<0/1>; Options: Enabled (1) / Disabled (0) (Default) + /// + [Newtonsoft.Json.JsonProperty(PropertyName = "useSystemTrustStore")] + public object UseSystemTrustStore {get; set; } + /// /// Gets or sets the Azure key vault secret reference of password in connection /// string. @@ -138,6 +172,8 @@ public virtual void Validate() + + if (this.Password != null) { this.Password.Validate(); diff --git a/src/DataFactory/DataFactory.Management.Sdk/README.md b/src/DataFactory/DataFactory.Management.Sdk/README.md index 37e3ab4c8a2c..f6a0712c84c8 100644 --- a/src/DataFactory/DataFactory.Management.Sdk/README.md +++ b/src/DataFactory/DataFactory.Management.Sdk/README.md @@ -23,7 +23,7 @@ payload-flattening-threshold: 2 ### ``` yaml -commit: 679e6bd33cc09224a02e73573c8818feb355eff8 +commit: 1ad29756bd141a47cac770140105a706d065ae1b require: https://github.com/Azure/azure-rest-api-specs/blob/$(commit)/specification/datafactory/resource-manager/readme.md output-folder: Generated diff --git a/src/DataFactory/DataFactory/ChangeLog.md b/src/DataFactory/DataFactory/ChangeLog.md index 18e78f5ac005..ce8cf3fb7e53 100644 --- a/src/DataFactory/DataFactory/ChangeLog.md +++ b/src/DataFactory/DataFactory/ChangeLog.md @@ -18,6 +18,8 @@ - Additional information about change #1 --> ## Upcoming Release +* Added support for Iceberg format as a sink. +* Enabled sslMode and useSystemTrustStore options for MariaDB. ## Version 1.18.7 * Supported managed identity for Data Factory Azure File connector.