Merge pull request #877 from gooddata/snapshot-master-d20db45e-to-rel…

…/dev

[bot] Merge master/d20db45e into rel/dev

.envrc

@@ -9,7 +9,7 @@ export PYTHONPATH="${PYTHONPATH}:${SCRIPT_DIR}/gooddata-api-client/"
 export PYTHONPATH="${PYTHONPATH}:${SCRIPT_DIR}/gooddata-sdk/"
 export PYTHONPATH="${PYTHONPATH}:${SCRIPT_DIR}/gooddata-pandas/"
 export PYTHONPATH="${PYTHONPATH}:${SCRIPT_DIR}/gooddata-flight-server/"
-export PYTHONPATH="${PYTHONPATH}:${SCRIPT_DIR}/gooddata-flexfun/"
+export PYTHONPATH="${PYTHONPATH}:${SCRIPT_DIR}/gooddata-flexconnect/"
 export PYTHONPATH="${PYTHONPATH}:${SCRIPT_DIR}/gooddata-dbt/"
 
 export PATH="${PATH}:${SCRIPT_DIR}/gooddata-sdk/bin"

.github/workflows/build-release.yaml

@@ -29,7 +29,7 @@ jobs:
           - gooddata-sdk
           - gooddata-dbt
           - gooddata-flight-server
-          - gooddata-flexfun
+          - gooddata-flexconnect
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
@@ -83,7 +83,7 @@ jobs:
           - gooddata-sdk
           - gooddata-dbt
           - gooddata-flight-server
-          - gooddata-flexfun
+          - gooddata-flexconnect
     steps:
       - name: Obtain ${{ matrix.component }} artifacts
         uses: actions/download-artifact@v4

.github/workflows/dev-release.yaml

@@ -21,7 +21,7 @@ jobs:
           - gooddata-sdk
           - gooddata-dbt
           - gooddata-flight-server
-          - gooddata-flexfun
+          - gooddata-flexconnect
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4

.github/workflows/rw-collect-changes.yaml

@@ -47,4 +47,4 @@ jobs:
               - 'gooddata-api-client/**'
               - 'gooddata-dbt/**'
               - 'gooddata-flight-server/**'
-              - 'gooddata-flexfun/**'
+              - 'gooddata-flexconnect/**'

.github/workflows/rw-python-tests.yaml

@@ -24,7 +24,7 @@ jobs:
         if: ${{ matrix.python_version == 'py312' }}
         uses: codecov/codecov-action@v3
         with:
-          files: ./gooddata-sdk/coverage.xml,./gooddata-pandas/coverage.xml,./gooddata-fdw/coverage.xml,./gooddata-flight-server/coverage.xml,./gooddata-flexfun/coverage.xml
+          files: ./gooddata-sdk/coverage.xml,./gooddata-pandas/coverage.xml,./gooddata-fdw/coverage.xml,./gooddata-flight-server/coverage.xml,./gooddata-flexconnect/coverage.xml
   lint-and-format-check:
     runs-on: ubuntu-latest
     if: ${{inputs.changed-python-modules == 'true'}}

.sonar.settings

@@ -1,4 +1,4 @@
 # (C) 2021 GoodData Corporation
-sonar.sources=gooddata-sdk,gooddata-fdw,gooddata-pandas,gooddata-flight-server,gooddata-flexfun
+sonar.sources=gooddata-sdk,gooddata-fdw,gooddata-pandas,gooddata-flight-server,gooddata-flexconnect
 sonar.exclusions=gooddata-api-client/**/*
 sonar.python.version=3.9, 3.10, 3.11, 3.12

.sonarcloud.properties

@@ -1,4 +1,4 @@
 # (C) 2022 GoodData Corporation
-sonar.sources=gooddata-sdk,gooddata-fdw,gooddata-pandas,gooddata-flight-server,gooddata-flexfun
+sonar.sources=gooddata-sdk,gooddata-fdw,gooddata-pandas,gooddata-flight-server,gooddata-flexconnect
 sonar.exclusions=gooddata-api-client/**/*
 sonar.python.version=3.9, 3.10, 3.11, 3.12

LICENSE.txt

@@ -7,4 +7,4 @@ The above copyright notice and this permission notice shall be included in all c
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 -----------------------
 
-This repository includes source code for gooddata-api-client, gooddata-fdw, gooddata-pandas, gooddata-sdk, gooddata-flight-server, gooddata-flexfun, and gooddata-dbt, each of which is licensed under the MIT license above. For additional licensing information related to open source packages included in each of these projects, please see the OSS LICENSES directory.
+This repository includes source code for gooddata-api-client, gooddata-fdw, gooddata-pandas, gooddata-sdk, gooddata-flight-server, gooddata-flexconnect, and gooddata-dbt, each of which is licensed under the MIT license above. For additional licensing information related to open source packages included in each of these projects, please see the OSS LICENSES directory.

dev-requirements.txt

@@ -10,11 +10,11 @@ pre-commit~=3.8.0
 -r ./gooddata-fdw/requirements.txt
 -r ./gooddata-dbt/requirements.txt
 -r ./gooddata-flight-server/requirements.txt
--r ./gooddata-flexfun/requirements.txt
+-r ./gooddata-flexconnect/requirements.txt
 -r ./gooddata-sdk/test-requirements.txt
 -r ./gooddata-pandas/test-requirements.txt
 -r ./gooddata-fdw/test-requirements.txt
 -r ./gooddata-dbt/test-requirements.txt
 -r ./gooddata-flight-server/test-requirements.txt
--r ./gooddata-flexfun/test-requirements.txt
+-r ./gooddata-flexconnect/test-requirements.txt
 -r ./tests-support/requirements.txt

gooddata-flexfun/.readthedocs.yaml → gooddata-flexconnect/.readthedocs.yaml

@@ -12,8 +12,8 @@ build:
     python: "3.11"
 
 sphinx:
-  configuration: gooddata-flexfun/docs/conf.py
+  configuration: gooddata-flexconnect/docs/conf.py
 
 python:
   install:
-    - requirements: gooddata-flexfun/docs/requirements.txt
+    - requirements: gooddata-flexconnect/docs/requirements.txt

gooddata-flexconnect/README.md

@@ -0,0 +1,61 @@
+# GoodData FlexConnect
+
+GoodData FlexConnect allows you to build your own data source for GoodData Cloud or Cloud Native.
+
+FlexConnect works with a concept similar to 'table functions' that you may already know
+from database technologies.
+
+-  To build your own data source, you implement one or more FlexConnect functions. The
+   functions compute and return tabular data - how they do it is completely up to you.
+-  The functions are hosted and invoked inside a FlexConnect server (which is included in this package).
+-  A running FlexConnect server can be added as a data source to your GoodData Cloud or GoodData Cloud Native.
+-  The functions available on FlexConnect server will be mapped to data sets within GoodData's Semantic Model
+   and from then on can be used during report computation.
+
+
+## Getting Started using the FlexConnect Template
+
+The easiest way to get started writing FlexConnect functions is to use [the template repository](https://github.com/gooddata/gooddata-flexconnect-template).
+It provides a simple example of a FlexConnect function that can be used as a starting point for your own FlexConnect functions with all the necessary infrastructure in place.
+It also has a README that explains how to get started with the template and some general tips on how to write FlexConnect functions.
+
+## Getting started using the FlexConnect package directly
+
+Install the package alongside the gooddata-flight-server using pip:
+
+```bash
+pip install gooddata-flight-server gooddata-flexconnect
+```
+
+Next, update the GoodData Flight Server configuration to load the FlexConnect functions.
+
+```toml
+[flexconnect]
+
+# specify one or more modules that contain your FlexConnect function implementations
+#
+functions = [
+    "flexconnect.your_function"
+]
+```
+
+Then when running the GoodData Flight Server, use the `--methods-provider` option to load the FlexConnect.
+For example:
+
+```bash
+#!/bin/bash
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+SERVER_CMD="${SCRIPT_DIR}/.venv/bin/gooddata-flight-server"
+
+export PYTHONPATH="${SCRIPT_DIR}/src"
+export CONFIG_ENV="${1:-dev}"
+
+$SERVER_CMD start \
+              --methods-provider gooddata_flexconnect \
+              --config \
+                config/${CONFIG_ENV}.server.toml \
+                config/flexconnect.config.toml \
+              --logging-config config/default.logging.ini \
+              --dev-log
+```

gooddata-flexfun/gooddata_flexfun/__init__.py → gooddata-flexconnect/gooddata_flexconnect/__init__.py

@@ -1,7 +1,6 @@
 # (C) 2024 GoodData Corporation
 
-from gooddata_flexfun.flexfun.flex_fun import FlexFun
-from gooddata_flexfun.flexfun.flex_fun_execution_context import (
+from gooddata_flexconnect.function.execution_context import (
     ExecutionContext,
     ExecutionContextAbsoluteDateFilter,
     ExecutionContextAttribute,
@@ -14,4 +13,5 @@
     LabelElementsExecutionRequest,
     ReportExecutionRequest,
 )
-from gooddata_flexfun.flexfun.flight_methods import create_flexfun_flight_methods
+from gooddata_flexconnect.function.flight_methods import create_flexconnect_flight_methods
+from gooddata_flexconnect.function.function import FlexConnectFunction

gooddata-flexfun/gooddata_flexfun/flexfun/flex_fun_execution_context.py → gooddata-flexconnect/gooddata_flexconnect/function/execution_context.py

@@ -236,7 +236,7 @@ class ExecutionContextAbsoluteDateFilter:
 @dataclass
 class ExecutionRequest:
     """
-    Information about the execution request that is sent to the FlexFun.
+    Information about the execution request that is sent to the FlexConnect function.
     DEPRECATED: Use ReportExecutionRequest instead.
     """
 
@@ -481,7 +481,7 @@ def _dict_to_attributes(attributes: list[dict]) -> list[ExecutionContextAttribut
 @dataclass
 class ExecutionContext:
     """
-    Execution context of the FlexFun
+    Execution context of the FlexConnect function
     """
 
     execution_type: ExecutionType
@@ -491,17 +491,17 @@ class ExecutionContext:
 
     organization_id: str
     """
-    The ID of the organization that the FlexFun is executed in.
+    The ID of the organization that the FlexConnect function is executed in.
     """
 
     workspace_id: str
     """
-    The ID of the workspace that the FlexFun is executed in.
+    The ID of the workspace that the FlexConnect function is executed in.
     """
 
     user_id: str
     """
-    The ID of the user that invoked the FlexFun.
+    The ID of the user that invoked the FlexConnect function.
     """
 
     timestamp: Optional[str]
@@ -532,13 +532,13 @@ class ExecutionContext:
 
     report_execution_request: Optional[ReportExecutionRequest]
     """
-    The report execution request that the FlexFun should process.
+    The report execution request that the FlexConnect function should process.
     Only present if the execution type is "REPORT".
     """
 
     label_elements_execution_request: Optional[LabelElementsExecutionRequest]
     """
-    The label elements execution request that the FlexFun should process.
+    The label elements execution request that the FlexConnect function should process.
     Only present if the execution type is "LABEL_ELEMENTS".
     """
 
@@ -568,8 +568,9 @@ def from_dict(d: dict) -> "ExecutionContext":
     @staticmethod
     def from_parameters(parameters: dict) -> Optional["ExecutionContext"]:
         """
-        Create ExecutionContext from FlexFun parameters.
-        :param parameters: the parameters dictionary of the FlexFun invocation
+        Create ExecutionContext from FlexConnect function parameters.
+
+        :param parameters: the parameters dictionary of the FlexConnect function invocation
         :return: None if the parameters do not contain the execution context, otherwise the execution context
         """
         return ExecutionContext.from_dict(parameters["executionContext"]) if "executionContext" in parameters else None

gooddata-flexfun/gooddata_flexfun/flexfun/flight_methods.py → gooddata-flexconnect/gooddata_flexconnect/function/flight_methods.py

@@ -16,16 +16,16 @@
     flight_server_methods,
 )
 
-from gooddata_flexfun.flexfun.flex_fun import FlexFun
-from gooddata_flexfun.flexfun.flex_fun_registry import FlexFunRegistry
-from gooddata_flexfun.flexfun.flex_fun_task import FlexFunTask
+from gooddata_flexconnect.function.function import FlexConnectFunction
+from gooddata_flexconnect.function.function_registry import FlexConnectFunctionRegistry
+from gooddata_flexconnect.function.function_task import FlexConnectFunctionTask
 
-_LOGGER = structlog.get_logger("gooddata_flexfun.rpc")
+_LOGGER = structlog.get_logger("gooddata_flexconnect.rpc")
 _DEFAULT_TASK_WAIT = 60.0
 
 
-class _FlexFunServerMethods(FlightServerMethods):
-    def __init__(self, ctx: ServerContext, registry: FlexFunRegistry) -> None:
+class _FlexConnectServerMethods(FlightServerMethods):
+    def __init__(self, ctx: ServerContext, registry: FlexConnectFunctionRegistry) -> None:
         self._ctx = ctx
         self._registry = registry
 
@@ -38,7 +38,7 @@ def _create_descriptor(fun_name: str, metadata: Optional[dict]) -> pyarrow.fligh
 
         return pyarrow.flight.FlightDescriptor.for_command(orjson.dumps(cmd))
 
-    def _create_fun_info(self, fun: type[FlexFun]) -> pyarrow.flight.FlightInfo:
+    def _create_fun_info(self, fun: type[FlexConnectFunction]) -> pyarrow.flight.FlightInfo:
         # these are for type checker; the registry will only register functions
         # that have proper metadata on them
         assert fun.Name is not None
@@ -57,18 +57,21 @@ def _extract_invocation_payload(
     ) -> tuple[str, dict, Optional[tuple[str, ...]]]:
         if descriptor.command is None or not len(descriptor.command):
             raise ErrorInfo.bad_argument(
-                "Incorrect FlexFun invocation. Flight descriptor must contain command with the invocation payload."
+                "Incorrect FlexConnect function invocation. Flight descriptor must contain command "
+                "with the invocation payload."
             )
 
         try:
             payload = orjson.loads(descriptor.command)
         except Exception:
-            raise ErrorInfo.bad_argument("Incorrect FlexFun invocation. The invocation payload is not a valid JSON.")
+            raise ErrorInfo.bad_argument(
+                "Incorrect FlexConnect function invocation. The invocation payload is " "not a valid JSON."
+            )
 
         fun = payload.get("functionName")
         if fun is None or not len(fun):
             raise ErrorInfo.bad_argument(
-                "Incorrect FlexFun invocation. The invocation payload does not specify 'functionName'."
+                "Incorrect FlexConnect function invocation. The invocation payload does not specify 'functionName'."
             )
 
         parameters = payload.get("parameters") or {}
@@ -80,13 +83,13 @@ def _prepare_task(
         self,
         context: pyarrow.flight.ServerCallContext,
         descriptor: pyarrow.flight.FlightDescriptor,
-    ) -> FlexFunTask:
+    ) -> FlexConnectFunctionTask:
         fun_name, parameters, columns = self._extract_invocation_payload(descriptor)
         headers = self.call_info_middleware(context).headers
-        flex_fun = self._registry.create_function(fun_name)
+        fun = self._registry.create_function(fun_name)
 
-        return FlexFunTask(
-            fun=flex_fun,
+        return FlexConnectFunctionTask(
+            fun=fun,
             parameters=parameters,
             columns=columns,
             headers=headers,
@@ -100,7 +103,7 @@ def _prepare_flight_info(self, task_result: TaskExecutionResult) -> pyarrow.flig
         if task_result.cancelled:
             raise ErrorInfo.for_reason(
                 ErrorCode.COMMAND_CANCELLED,
-                f"FlexFun invocation was cancelled. Invocation task was: '{task_result.task_id}'.",
+                f"FlexConnect function invocation was cancelled. Invocation task was: '{task_result.task_id}'.",
             ).to_server_error()
 
         result = task_result.result
@@ -127,17 +130,17 @@ def list_flights(
         self, context: pyarrow.flight.ServerCallContext, criteria: bytes
     ) -> Generator[pyarrow.flight.FlightInfo, None, None]:
         structlog.contextvars.bind_contextvars(peer=context.peer())
-        _LOGGER.info("list_flights", available_funs=self._registry.flex_funs_names)
+        _LOGGER.info("list_flights", available_funs=self._registry.function_names)
 
-        return (self._create_fun_info(fun) for fun in self._registry.flex_funs.values())
+        return (self._create_fun_info(fun) for fun in self._registry.functions.values())
 
     def get_flight_info(
         self,
         context: pyarrow.flight.ServerCallContext,
         descriptor: pyarrow.flight.FlightDescriptor,
     ) -> pyarrow.flight.FlightInfo:
         structlog.contextvars.bind_contextvars(peer=context.peer())
-        task: Optional[FlexFunTask] = None
+        task: Optional[FlexConnectFunctionTask] = None
 
         try:
             task = self._prepare_task(context, descriptor)
@@ -163,7 +166,7 @@ def get_flight_info(
             if task is not None:
                 _LOGGER.error("get_flight_info_failed", task_id=task.task_id, fun=task.fun_name, exc_info=True)
             else:
-                _LOGGER.error("flexfun_submit_failed", exc_info=True)
+                _LOGGER.error("flexconnect_fun_submit_failed", exc_info=True)
 
             raise
 
@@ -190,23 +193,23 @@ def do_get(
             raise
 
 
-_FLEXFUN_CONFIG_SECTION = "flexfun"
-_FLEXFUN_FUNCTION_LIST = "functions"
+_FLEX_CONNECT_CONFIG_SECTION = "flexconnect"
+_FLEX_CONNECT_FUNCTION_LIST = "functions"
 
 
 @flight_server_methods
-def create_flexfun_flight_methods(ctx: ServerContext) -> FlightServerMethods:
+def create_flexconnect_flight_methods(ctx: ServerContext) -> FlightServerMethods:
     """
-    This factory creates implementation of Flight RPC methods that realize the FlexFun server.
+    This factory creates implementation of Flight RPC methods that realize the FlexConnect server.
 
-    FlexFun Server hosts one or more functions developed externally, and linked to the server
+    FlexConnect Server hosts one or more functions developed externally, and linked to the server
     at runtime - during startup.
 
     :param ctx: server's context
     :return: new instance of Flight RPC server methods to integrate into the server
     """
-    modules = list(ctx.settings.get(f"{_FLEXFUN_CONFIG_SECTION}.{_FLEXFUN_FUNCTION_LIST}") or [])
-    _LOGGER.info("flexfun_init", modules=modules)
-    registry = FlexFunRegistry().load(ctx, modules)
+    modules = list(ctx.settings.get(f"{_FLEX_CONNECT_CONFIG_SECTION}.{_FLEX_CONNECT_FUNCTION_LIST}") or [])
+    _LOGGER.info("flexconnect_init", modules=modules)
+    registry = FlexConnectFunctionRegistry().load(ctx, modules)
 
-    return _FlexFunServerMethods(ctx, registry)
+    return _FlexConnectServerMethods(ctx, registry)