feat(source-linkedin-ads): move custom streams to manifest only

airbyte-integrations/connectors/source-linkedin-ads/metadata.yaml

@@ -11,7 +11,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 137ece28-5434-455c-8f34-69dc3782f451
-  dockerImageTag: 5.0.0
+  dockerImageTag: 5.1.0
   dockerRepository: airbyte/source-linkedin-ads
   documentationUrl: https://docs.airbyte.com/integrations/sources/linkedin-ads
   githubIssueLabel: source-linkedin-ads

airbyte-integrations/connectors/source-linkedin-ads/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "5.0.0"
+version = "5.1.0"
 name = "source-linkedin-ads"
 description = "Source implementation for Linkedin Ads."
 authors = [ "Airbyte <[email protected]>",]
@@ -17,7 +17,7 @@ include = "source_linkedin_ads"
 
 [tool.poetry.dependencies]
 python = "^3.10,<3.12"
-airbyte-cdk = "^5"
+airbyte-cdk = { version = "6.12.1.dev0", allow-prereleases = true }
 coverage = "^7.5.3"
 
 [tool.poetry.scripts]

airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/manifest.yaml

@@ -391,15 +391,15 @@ definitions:
         - sponsoredCampaign
       retriever:
         class_name: "source_linkedin_ads.components.LinkedInAdsCustomRetriever"
+        $parameters:
+          name: ad_campaign_analytics
         requester:
           type: CustomRequester
           class_name: "source_linkedin_ads.components.SafeEncodeHttpRequester"
           url_base: https://api.linkedin.com/rest/
           authenticator: "#/definitions/authenticator"
           path: adAnalytics
           http_method: GET
-          $parameters:
-            name: ad_campaign_analytics
           request_parameters:
             q: analytics
             pivot: "(value:CAMPAIGN)"
@@ -466,15 +466,15 @@ definitions:
         - end_date
       retriever:
         class_name: "source_linkedin_ads.components.LinkedInAdsCustomRetriever"
+        $parameters:
+          name: ad_creative_analytics
         requester:
           type: CustomRequester
           class_name: "source_linkedin_ads.components.SafeEncodeHttpRequester"
           url_base: https://api.linkedin.com/rest/
           authenticator: "#/definitions/authenticator"
           path: adAnalytics
           http_method: GET
-          $parameters:
-            name: ad_creative_analytics
           request_parameters:
             q: analytics
             pivot: "(value:CREATIVE)"
@@ -541,15 +541,15 @@ definitions:
         - end_date
       retriever:
         class_name: "source_linkedin_ads.components.LinkedInAdsCustomRetriever"
+        $parameters:
+          name: ad_impression_device_analytics
         requester:
           type: CustomRequester
           class_name: "source_linkedin_ads.components.SafeEncodeHttpRequester"
           url_base: https://api.linkedin.com/rest/
           authenticator: "#/definitions/authenticator"
           path: adAnalytics
           http_method: GET
-          $parameters:
-            name: ad_impression_device_analytics
           request_parameters:
             q: analytics
             pivot: "(value:IMPRESSION_DEVICE_TYPE)"
@@ -616,15 +616,15 @@ definitions:
         - end_date
       retriever:
         class_name: "source_linkedin_ads.components.LinkedInAdsCustomRetriever"
+        $parameters:
+          name: ad_member_company_size_analytics
         requester:
           type: CustomRequester
           class_name: "source_linkedin_ads.components.SafeEncodeHttpRequester"
           url_base: https://api.linkedin.com/rest/
           authenticator: "#/definitions/authenticator"
           path: adAnalytics
           http_method: GET
-          $parameters:
-            name: ad_member_company_size_analytics
           request_parameters:
             q: analytics
             pivot: "(value:MEMBER_COMPANY_SIZE)"
@@ -691,15 +691,15 @@ definitions:
         - end_date
       retriever:
         class_name: "source_linkedin_ads.components.LinkedInAdsCustomRetriever"
+        $parameters:
+          name: ad_member_country_analytics
         requester:
           type: CustomRequester
           class_name: "source_linkedin_ads.components.SafeEncodeHttpRequester"
           url_base: https://api.linkedin.com/rest/
           authenticator: "#/definitions/authenticator"
           path: adAnalytics
           http_method: GET
-          $parameters:
-            name: ad_member_country_analytics
           request_parameters:
             q: analytics
             pivot: "(value:MEMBER_COUNTRY_V2)"
@@ -766,15 +766,15 @@ definitions:
         - end_date
       retriever:
         class_name: "source_linkedin_ads.components.LinkedInAdsCustomRetriever"
+        $parameters:
+          name: ad_member_job_function_analytics
         requester:
           type: CustomRequester
           class_name: "source_linkedin_ads.components.SafeEncodeHttpRequester"
           url_base: https://api.linkedin.com/rest/
           authenticator: "#/definitions/authenticator"
           path: adAnalytics
           http_method: GET
-          $parameters:
-            name: ad_member_job_function_analytics
           request_parameters:
             q: analytics
             pivot: "(value:MEMBER_JOB_FUNCTION)"
@@ -841,15 +841,15 @@ definitions:
         - end_date
       retriever:
         class_name: "source_linkedin_ads.components.LinkedInAdsCustomRetriever"
+        $parameters:
+          name: ad_member_job_title_analytics
         requester:
           type: CustomRequester
           class_name: "source_linkedin_ads.components.SafeEncodeHttpRequester"
           url_base: https://api.linkedin.com/rest/
           authenticator: "#/definitions/authenticator"
           path: adAnalytics
           http_method: GET
-          $parameters:
-            name: ad_member_job_title_analytics
           request_parameters:
             q: analytics
             pivot: "(value:MEMBER_JOB_TITLE)"
@@ -916,15 +916,15 @@ definitions:
         - end_date
       retriever:
         class_name: "source_linkedin_ads.components.LinkedInAdsCustomRetriever"
+        $parameters:
+          name: ad_member_industry_analytics
         requester:
           type: CustomRequester
           class_name: "source_linkedin_ads.components.SafeEncodeHttpRequester"
           url_base: https://api.linkedin.com/rest/
           authenticator: "#/definitions/authenticator"
           path: adAnalytics
           http_method: GET
-          $parameters:
-            name: ad_member_industry_analytics
           request_parameters:
             q: analytics
             pivot: "(value:MEMBER_INDUSTRY)"
@@ -991,15 +991,15 @@ definitions:
         - end_date
       retriever:
         class_name: "source_linkedin_ads.components.LinkedInAdsCustomRetriever"
+        $parameters:
+          name: ad_member_seniority_analytics
         requester:
           type: CustomRequester
           class_name: "source_linkedin_ads.components.SafeEncodeHttpRequester"
           url_base: https://api.linkedin.com/rest/
           authenticator: "#/definitions/authenticator"
           path: adAnalytics
           http_method: GET
-          $parameters:
-            name: ad_member_seniority_analytics
           request_parameters:
             q: analytics
             pivot: "(value:MEMBER_SENIORITY)"
@@ -1066,15 +1066,15 @@ definitions:
         - end_date
       retriever:
         class_name: "source_linkedin_ads.components.LinkedInAdsCustomRetriever"
+        $parameters:
+          name: ad_member_region_analytics
         requester:
           type: CustomRequester
           class_name: "source_linkedin_ads.components.SafeEncodeHttpRequester"
           url_base: https://api.linkedin.com/rest/
           authenticator: "#/definitions/authenticator"
           path: adAnalytics
           http_method: GET
-          $parameters:
-            name: ad_member_region_analytics
           request_parameters:
             q: analytics
             pivot: "(value:MEMBER_REGION_V2)"
@@ -1141,15 +1141,15 @@ definitions:
         - end_date
       retriever:
         class_name: "source_linkedin_ads.components.LinkedInAdsCustomRetriever"
+        $parameters:
+          name: ad_member_company_analytics
         requester:
           type: CustomRequester
           class_name: "source_linkedin_ads.components.SafeEncodeHttpRequester"
           url_base: https://api.linkedin.com/rest/
           authenticator: "#/definitions/authenticator"
           path: adAnalytics
           http_method: GET
-          $parameters:
-            name: ad_member_company_analytics
           request_parameters:
             q: analytics
             pivot: "(value:MEMBER_COMPANY)"
@@ -1208,6 +1208,82 @@ definitions:
         type: InlineSchemaLoader
         schema:
           $ref: "#/schemas/ad_member_company_analytics"
+    custom_analytics_report:
+      type: DeclarativeStream
+      name: custom_analytics_report
+      primary_key:
+        - string_of_pivot_values
+        - end_date
+        - sponsoredCampaign
+      retriever:
+        class_name: "source_linkedin_ads.components.LinkedInAdsCustomRetriever"
+        $parameters:
+          name: custom_analytics_report
+        requester:
+          type: CustomRequester
+          class_name: "source_linkedin_ads.components.SafeEncodeHttpRequester"
+          url_base: https://api.linkedin.com/rest/
+          authenticator: "#/definitions/authenticator"
+          path: adAnalytics
+          http_method: GET
+          request_parameters:
+            q: analytics
+            pivot: "DYNAMIC_FIELD"
+            timeGranularity: "DYNAMIC_FIELD"
+            campaigns: "List(urn%3Ali%3AsponsoredCampaign%3A{{stream_slice.get('campaign_id')}})"
+            dateRange: "(start:(year:{{stream_slice.get('start.year')}},month:{{stream_slice.get('start.month')}},day:{{stream_slice.get('start.day')}}),end:(year:{{stream_slice.get('end.year')}},month:{{stream_slice.get('end.month')}},day:{{stream_slice.get('end.day')}}))"
+            fields: "{{stream_slice['fields']}}"
+          request_headers:
+            Linkedin-Version: "202404"
+            X-Restli-Protocol-Version: 2.0.0
+        record_selector:
+          type: RecordSelector
+          $parameters:
+            transformations:
+              - type: AddFields
+                fields:
+                  - path:
+                      - sponsoredCampaign
+                    value: "{{stream_slice.get('campaign_id')}}"
+                    value_type: string
+              - type: AddFields
+                fields:
+                  - path:
+                      - pivot
+                    value: "DYNAMIC_FIELD"
+                    value_type: string
+          extractor:
+            type: CustomRecordExtractor
+            class_name: source_linkedin_ads.components.LinkedInAdsRecordExtractor
+          schema_normalization: Default
+        paginator:
+          type: NoPagination
+        partition_router:
+          type: SubstreamPartitionRouter
+          parent_stream_configs:
+            - type: ParentStreamConfig
+              parent_key: id
+              partition_field: campaign_id
+              stream:
+                $ref: "#/definitions/streams/campaigns"
+      incremental_sync:
+        type: CustomIncrementalSync
+        class_name: source_linkedin_ads.components.AnalyticsDatetimeBasedCursor
+        cursor_field: end_date
+        cursor_datetime_formats:
+          - "%Y-%m-%d"
+        datetime_format: "%Y-%m-%d"
+        cursor_granularity: P0D
+        step: P30D
+        lookback_window: P{{ config.get('lookback_window', 0) }}D
+        start_datetime:
+          type: MinMaxDatetime
+          datetime: "{{ config['start_date'] }}"
+          datetime_format: "%Y-%m-%d"
+      schema_loader:
+        type: InlineSchemaLoader
+        schema:
+          $ref: "#/schemas/ad_campaign_analytics"
   base_requester:
     type: HttpRequester
     url_base: https://api.linkedin.com/rest/
@@ -1249,6 +1325,40 @@ streams:
   - $ref: "#/definitions/streams/ad_member_region_analytics"
   - $ref: "#/definitions/streams/ad_member_company_analytics"
 
+dynamic_streams:
+  - type: DynamicDeclarativeStream
+    stream_template:
+      $ref: "#/definitions/streams/custom_analytics_report"
+    components_resolver:
+      type: ConfigComponentsResolver
+      stream_config:
+        type: StreamConfig
+        configs_pointer:
+          - ad_analytics_reports
+      components_mapping:
+        - type: ComponentMappingDefinition
+          field_path: ["**", "name"]
+          value: "custom_{{components_values['name']}}"
+        - type: ComponentMappingDefinition
+          field_path: ["retriever", "requester", "request_parameters", "pivot"]
+          value: "(value:{{components_values['pivot_by']}})"
+        - type: ComponentMappingDefinition
+          field_path:
+            ["retriever", "requester", "request_parameters", "timeGranularity"]
+          value: "(value:{{components_values['time_granularity']}})"
+        - type: ComponentMappingDefinition
+          field_path:
+            [
+              "retriever",
+              "record_selector",
+              "**",
+              "transformations",
+              1,
+              "fields",
+              0,
+              "value",
+            ]
+          value: "{{components_values['pivot_by']}}"
 spec:
   type: Spec
   connection_specification: