Merge branch 'master' into source/scaping

.github/pull_request_template.md

@@ -1,12 +1,30 @@
-# Tell us what you do here
+<!--
+Thank you for submitting a pull request! Please provide a brief description of your changes below.
+-->
+### Tell us what you do here
+<!--
+Pick the relevant item or items and remove the rest: 
+-->
+- implementing verified source (please link a relevant issue labeled as `verified source`)
+- fixing a bug (please link a relevant bug report)
+- improving, documenting, or customizing an existing source (please link an issue or describe below)
+- anything else (please link an issue or describe below)
 
-- [ ] implementing verified source (please link a relevant issue labeled as `verified source`)
-- [ ] fixing a bug (please link a relevant bug report)
-- [ ] improving, documenting, or customizing an existing source (please link an issue or describe below)
-- [ ] anything else (please link an issue or describe below)
+### Short description
 
-# Relevant issue
+<!--
+Please link any related issues. This helps us keep the PR focused and merge it faster.
+-->
+### Related Issues
 
-issue #
+- Fixes #...
+- Closes #...
+- Resolves #...
 
-# More PR info
+### Additional Context
+
+<!--
+Please ensure that
+    - you have read the [Contributing Guide](../CONTRIBUTING.md) guide.
+    - you have run the tests locally and they have passed before submitting your PR.
+-->

.github/workflows/get_changed_sources.yml

@@ -28,7 +28,7 @@ jobs:
 
           raw_changed_sources=$(
             git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} \
-              | grep '^sources/[^._]' || true
+              | grep -E '^sources/[^._]|^tests/[^._]' || true
           )
 
           echo $raw_changed_sources
@@ -37,10 +37,11 @@ jobs:
             echo "No changed sources. Skipping tests."
           else
             changed_sources=$(echo "$raw_changed_sources" \
-              | sed -nE 's/^sources\/([^/]+)\/.*$/\1/p; s/^sources\/(.*)_pipeline\.py$/\1/p' \
+              | sed -nE 's/^sources\/([^/]+)\/.*$/\1/p; s/^sources\/(.*)_pipeline\.py$/\1/p; s/^tests\/([^/]+)\/.*$/\1/p' \
               | sort -u \
               | tr '\n' ' '
             )
-            echo "Changed sources: $changed_sources"
+            echo "Changed sources or sources tests: $changed_sources"
           fi
+
           echo "sources_list=$changed_sources" >> $GITHUB_OUTPUT

pyproject.toml

@@ -12,7 +12,7 @@ packages = [{include = "sources"}]
 
 [tool.poetry.dependencies]
 python = ">=3.8.1,<3.13"
-dlt = {version = "0.4.4", allow-prereleases = true, extras = ["redshift", "bigquery", "postgres", "duckdb", "s3", "gs"]}
+dlt = {version = "0.4.5", allow-prereleases = true, extras = ["redshift", "bigquery", "postgres", "duckdb", "s3", "gs"]}
 
 [tool.poetry.group.dev.dependencies]
 mypy = "1.6.1"

sources/facebook_ads/__init__.py

@@ -1,17 +1,13 @@
 """Loads campaigns, ads sets, ads, leads and insight data from Facebook Marketing API"""
 
-import functools
-from typing import Any, Iterator, Sequence
-import time
+from typing import Iterator, Sequence
 
 from facebook_business import FacebookAdsApi
 from facebook_business.api import FacebookResponse
 
 import dlt
-from dlt.common import pendulum, logger
+from dlt.common import pendulum
 from dlt.common.typing import TDataItems, TDataItem, DictStrAny
-from dlt.common.time import ensure_pendulum_datetime
-from dlt.extract.typing import ItemTransformFunctionWithMeta
 from dlt.sources import DltResource
 
 from .helpers import (

sources/facebook_ads/helpers.py

@@ -13,7 +13,6 @@
 from dlt.common.configuration.inject import with_config
 from dlt.common.time import ensure_pendulum_datetime
 from dlt.common.typing import DictStrAny, TDataItem, TDataItems
-from dlt.extract.typing import ItemTransformFunctionWithMeta
 from dlt.sources.helpers import requests
 from dlt.sources.helpers.requests import Client
 
@@ -88,9 +87,7 @@ def get_data_chunked(
         yield chunk
 
 
-def enrich_ad_objects(
-    fb_obj_type: AbstractObject, fields: Sequence[str]
-) -> ItemTransformFunctionWithMeta[TDataItems]:
+def enrich_ad_objects(fb_obj_type: AbstractObject, fields: Sequence[str]) -> Any:
     """Returns a transformation that will enrich any of the resources returned by `` with additional fields
 
     In example below we add "thumbnail_url" to all objects loaded by `ad_creatives` resource:

sources/pipedrive/__init__.py

@@ -20,8 +20,7 @@
 from .settings import ENTITY_MAPPINGS, RECENTS_ENTITIES
 from dlt.common import pendulum
 from dlt.common.time import ensure_pendulum_datetime
-from dlt.sources import DltResource
-from dlt.extract.typing import DataItemWithMeta
+from dlt.sources import DltResource, TDataItems
 
 
 @dlt.source(name="pipedrive")
@@ -99,7 +98,7 @@ def pipedrive_source(
 
 def _get_deals_flow(
     deals_page: TDataPage, pipedrive_api_key: str
-) -> Iterator[DataItemWithMeta]:
+) -> Iterator[TDataItems]:
     custom_fields_mapping = dlt.current.source_state().get("custom_fields_mapping", {})
     for row in deals_page:
         url = f"deals/{row['id']}/flow"