Merge pull request #8 from tgrandje/dev

Create doc & diverse bugfixes/enhancements
tgrandje · Aug 8, 2024 · d4e93d2 · d4e93d2
2 parents 080ebb2 + f2dfbf4
commit d4e93d2
Show file tree

Hide file tree

Showing 29 changed files with 2,433 additions and 556 deletions.
diff --git a/.github/workflows/build_pages.yml b/.github/workflows/build_pages.yml
@@ -0,0 +1,71 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+# Sample workflow for building and deploying a Jekyll site to GitHub Pages
+name: Deploy Jekyll site to Pages
+
+on:
+  push:
+    branches:
+      - "main"
+    paths:
+      - "docs/**"
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow one concurrent deployment
+concurrency:
+  group: "pages"
+  cancel-in-progress: true
+
+jobs:
+  # Build job
+  build:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: docs
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Setup Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.3'
+          bundler-cache: true
+          cache-version: 0
+          working-directory: '${{ github.workspace }}/docs'
+      - name: Setup Pages
+        id: pages
+        uses: actions/configure-pages@v3
+      - name: Build with Jekyll
+        # Outputs to the './_site' directory by default
+        run: bundle exec jekyll build --baseurl "${{ steps.pages.outputs.base_path }}"
+        env:
+          JEKYLL_ENV: production
+      - name: Upload artifact
+        # Automatically uploads an artifact from the './_site' directory by default
+        uses: actions/upload-pages-artifact@v1
+        with:
+          path: "docs/_site/"
+
+  # Deployment job
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v2
diff --git a/cl_hubeau/drinking_water_quality/utils.py b/cl_hubeau/drinking_water_quality/utils.py
@@ -36,7 +36,7 @@ def get_all_water_networks(**kwargs) -> pd.DataFrame:
 
     """
 
-    city_codes = get_cities().tolist()
+    city_codes = get_cities()
 
     # Split by 20-something chunks
     city_codes = [
@@ -157,11 +157,11 @@ def get_control_results(
     return results
 
 
-if __name__ == "__main__":
-    df = get_control_results(
-        codes_communes="59350",
-        code_parametre="1340",
-        date_min_prelevement="2023-01-01",
-        date_max_prelevement="2023-12-31",
-    )
-    print(df)
+# if __name__ == "__main__":
+#     df = get_control_results(
+#         codes_communes="59350",
+#         code_parametre="1340",
+#         date_min_prelevement="2023-01-01",
+#         date_max_prelevement="2023-12-31",
+#     )
+#     print(df)
diff --git a/cl_hubeau/hydrometry/utils.py b/cl_hubeau/hydrometry/utils.py
@@ -33,7 +33,7 @@ def get_all_stations(**kwargs) -> gpd.GeoDataFrame:
 
     """
 
-    deps = get_departements().unique().tolist()
+    deps = get_departements()
 
     with HydrometrySession() as session:
         results = [
@@ -75,7 +75,7 @@ def get_all_sites(**kwargs) -> gpd.GeoDataFrame:
 
     """
 
-    deps = get_departements().unique().tolist()
+    deps = get_departements()
 
     with HydrometrySession() as session:
         results = [

diff --git a/cl_hubeau/piezometry/piezometry_scraper.py b/cl_hubeau/piezometry/piezometry_scraper.py
@@ -199,12 +199,7 @@ def get_realtime_chronicles(self, **kwargs):
                 kwargs.pop("code_bss"), 200
             )
         except KeyError:
-            # reset to default hubeau value, which is set even when code_bss is
-            # missing
-            code_bss = "07548X0009/F"
-            msg = f"code_bss is missing, will be set to {code_bss=} by hubeau"
-            logging.warning(msg)
-            params["code_bss"] = code_bss
+            pass
 
         try:
             params["bss_id"] = self.list_to_str_param(

diff --git a/cl_hubeau/piezometry/utils.py b/cl_hubeau/piezometry/utils.py
@@ -33,7 +33,7 @@ def get_all_stations(**kwargs) -> gpd.GeoDataFrame:
 
     with PiezometrySession() as session:
 
-        deps = get_departements().unique().tolist()
+        deps = get_departements()
         results = [
             session.get_stations(
                 code_departement=dep, format="geojson", **kwargs
@@ -93,19 +93,25 @@ def get_chronicles(codes_bss: list, **kwargs) -> pd.DataFrame:
     return results
 
 
-def get_realtime_chronicles(codes_bss: list, **kwargs) -> pd.DataFrame:
+def get_realtime_chronicles(
+    codes_bss: list = None, bss_ids: list = None, **kwargs
+) -> pd.DataFrame:
     """
     Retrieve realtimes chronicles from multiple piezometers.
     Uses a reduced timeout for cache expiration.
 
+    Note that `codes_bss` and `bss_ids` are mutually exclusive!
+
     Parameters
     ----------
-    codes_bss : list
-        List of code_bss codes for piezometers
+    codes_bss : list, optional
+        List of code_bss codes for piezometers. The default is None.
+    bss_ids : list, optional
+        List of bss_id codes for piezometers. The default is None.
     **kwargs :
-        kwargs passed to PiezometrySession.get_chronicles (hence mostly
-        intended for hub'eau API's arguments). Do not use `code_bss` as they
-        are set by the current function.
+        kwargs passed to PiezometrySession.get_realtime_chronicles (hence
+        mostly intended for hub'eau API's arguments). Do not use `code_bss` as
+        they are set by the current function.
 
     Returns
     -------
@@ -114,13 +120,25 @@ def get_realtime_chronicles(codes_bss: list, **kwargs) -> pd.DataFrame:
 
     """
 
+    if codes_bss and bss_ids:
+        raise ValueError(
+            "only one argument allowed among codes_bss and bss_ids"
+        )
+    if not codes_bss and not bss_ids:
+        raise ValueError(
+            "exactly one argument must be set among codes_bss and bss_ids"
+        )
+
+    code_names = "code_bss" if codes_bss else "bss_id"
+    codes = codes_bss if codes_bss else bss_ids
+
     with PiezometrySession(
         expire_after=_config["DEFAULT_EXPIRE_AFTER_REALTIME"]
     ) as session:
         results = [
-            session.get_realtime_chronicles(code_bss=code, **kwargs)
+            session.get_realtime_chronicles(**{code_names: code}, **kwargs)
             for code in tqdm(
-                codes_bss,
+                codes,
                 desc="querying piezo/piezo",
                 leave=_config["TQDM_LEAVE"],
                 position=tqdm._get_free_pos(),
@@ -129,8 +147,3 @@ def get_realtime_chronicles(codes_bss: list, **kwargs) -> pd.DataFrame:
     results = [x.dropna(axis=1, how="all") for x in results if not x.empty]
     results = pd.concat(results, ignore_index=True)
     return results
-
-
-if __name__ == "__main__":
-
-    df = get_all_stations()
diff --git a/cl_hubeau/session/session.py b/cl_hubeau/session/session.py
@@ -203,7 +203,7 @@ def list_to_str_param(
                 )
                 raise ValueError(msg)
 
-            return ",".join(x)
+            return ",".join([str(y) for y in x])
         if isinstance(x, str):
             return x
         raise ValueError(f"unexpected format for {x}")

diff --git a/cl_hubeau/utils/cities_and_departments.py b/cl_hubeau/utils/cities_and_departments.py
@@ -5,34 +5,134 @@
 """
 
 import os
-from pathlib import Path
 
 from pynsee import get_area_list
 from pynsee.utils.init_conn import init_conn
 
-from cl_hubeau.utils.clean_cache import clean_all_cache
-
 
 def init_pynsee_connection():
     """
     Initiate an INSEE API connection with tokens and proxies.
     """
-    home = str(Path.home())
-    pynsee_credentials_file = os.path.join(home, "pynsee_credentials.csv")
-    if not os.path.exists(pynsee_credentials_file):
-        clean_all_cache()
-        keys = ["insee_key", "insee_secret", "http_proxy", "https_proxy"]
-        kwargs = {x: os.environ[x] for x in keys if x in os.environ}
-        init_conn(**kwargs)
+    keys = ["insee_key", "insee_secret", "http_proxy", "https_proxy"]
+    kwargs = {x: os.environ[x] for x in keys if x in os.environ}
+    init_conn(**kwargs)
 
 
 def get_cities():
     init_pynsee_connection()
     cities = get_area_list("communes", "*", silent=True)
-    return cities["CODE"]
+    return cities["CODE"].unique().tolist()
 
 
 def get_departements():
-    init_pynsee_connection()
-    deps = get_area_list("departements", "*", silent=True)
-    return deps["CODE"]
+    try:
+        init_pynsee_connection()
+        deps = get_area_list("departements", "*", silent=True)
+        return deps["CODE"].unique().tolist()
+    except Exception:
+        # pynsee not working, return a simple constant
+        return [
+            "01",
+            "02",
+            "03",
+            "04",
+            "05",
+            "06",
+            "07",
+            "08",
+            "09",
+            "10",
+            "11",
+            "12",
+            "13",
+            "14",
+            "15",
+            "16",
+            "17",
+            "18",
+            "19",
+            "20",
+            "21",
+            "22",
+            "23",
+            "24",
+            "25",
+            "26",
+            "27",
+            "28",
+            "29",
+            "2A",
+            "2B",
+            "30",
+            "31",
+            "32",
+            "33",
+            "34",
+            "35",
+            "36",
+            "37",
+            "38",
+            "39",
+            "40",
+            "41",
+            "42",
+            "43",
+            "44",
+            "45",
+            "46",
+            "47",
+            "48",
+            "49",
+            "50",
+            "51",
+            "52",
+            "53",
+            "54",
+            "55",
+            "56",
+            "57",
+            "58",
+            "59",
+            "60",
+            "61",
+            "62",
+            "63",
+            "64",
+            "65",
+            "66",
+            "67",
+            "68",
+            "69",
+            "70",
+            "71",
+            "72",
+            "73",
+            "74",
+            "75",
+            "76",
+            "77",
+            "78",
+            "79",
+            "80",
+            "81",
+            "82",
+            "83",
+            "84",
+            "85",
+            "86",
+            "87",
+            "88",
+            "89",
+            "90",
+            "91",
+            "92",
+            "93",
+            "94",
+            "95",
+            "971",
+            "972",
+            "973",
+            "974",
+            "976",
+        ]
diff --git a/docs/.gitignore b/docs/.gitignore
@@ -0,0 +1,5 @@
+_site
+.sass-cache
+.jekyll-cache
+.jekyll-metadata
+vendor