From f94861cbfd8429a2f52d1d50c847d4274b968827 Mon Sep 17 00:00:00 2001 From: GeoWill Date: Thu, 11 Jul 2024 09:10:35 +0100 Subject: [PATCH] WiP --- .gitignore | 3 + README.md | 6 +- .../handler.py | 58 ++++ .../requirements.in | 2 + .../requirements.txt | 40 +++ cdk/queries/partition-addressbase-cleaned.sql | 9 + cdk/queries/uprn-to-ballots-first-letter.sql | 80 ++--- cdk/stacks/data_baker.py | 290 +++++++++++++++++- requirements/base.in | 1 + requirements/base.txt | 19 +- requirements/dev.in | 5 +- requirements/dev.txt | 135 ++++---- 12 files changed, 524 insertions(+), 124 deletions(-) create mode 100644 cdk/lambdas/partition-addressbase-by-first-letter/handler.py create mode 100644 cdk/lambdas/partition-addressbase-by-first-letter/requirements.in create mode 100644 cdk/lambdas/partition-addressbase-by-first-letter/requirements.txt create mode 100644 cdk/queries/partition-addressbase-cleaned.sql diff --git a/.gitignore b/.gitignore index 7b2a528..74abb45 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,6 @@ node_modules # CDK cdk.out + +# Python +**/__pycache__/** diff --git a/README.md b/README.md index 1188b23..c84b688 100644 --- a/README.md +++ b/README.md @@ -14,8 +14,8 @@ Use [`uv`](https://github.com/astral-sh/uv) or [`pip-tools`](https://github.com/ Commands given using `uv`. See [`uv` equivalence]() for comparison. ```shell -uv venv --prompt . .venv -uv pip sync requirements/dev.txt +PYENV_VERSION=3.10.14 uv venv --prompt . .venv +uv pip sync requirements/base.txt requirements/dev.txt ``` ### Commit hooks @@ -66,7 +66,7 @@ npm install #### dev dependencies * Add a dependency to `requirements/dev.in` -* Run `uv pip compile --generate-hashes --output-file requirements/dev.txt requirements/base.in requirements/dev.in` +* Run `uv pip compile --generate-hashes --output-file requirements/dev.txt requirements/dev.in` ## CDK Setup diff --git a/cdk/lambdas/partition-addressbase-by-first-letter/handler.py b/cdk/lambdas/partition-addressbase-by-first-letter/handler.py new file mode 100644 index 0000000..6f67312 --- /dev/null +++ b/cdk/lambdas/partition-addressbase-by-first-letter/handler.py @@ -0,0 +1,58 @@ +import boto3 + + +def handler(event, context): + if len(event["Records"]) != 1: + return {"message": "More than one object creation event sent"} + record = event["Records"][0] + if record["eventSource"] != "aws:s3": + return {"message": "Not an s3 Event"} + + if not record["eventName"].startswith("ObjectCreated"): + return {"message": "Not an object creation event"} + + # Do checks on addressbase_bucket and key?? + # addressbase_bucket = record["s3"]["addressbase_bucket"]["name"] + # key = record["s3"]["key"] + + # Invoke query + athena_client = boto3.client("athena") + workgroup = "dc-data-baker" + database = "dc_data_baker" + + query_id = get_query( + athena_client, + "partition-addressbase-cleaned-query", + workgroup, + ) + return athena_client.start_query_execution( + QueryExecutionId=query_id, + QueryExecutionContext={"Database": database}, + WorkGroup=workgroup, + ) + + # Update glue table description after query has run - this should probably be another lambda... + + +def get_query(client, name, workgroup): + query_ids = client.list_named_queries(WorkGroup=workgroup)["NamedQueryIds"] + for qid in query_ids: + query = client.get_named_query(NamedQueryId=qid) + if name == query["NamedQuery"]["Name"]: + return query + return None + + +# {'NamedQuery': {'Name': 'partition-addressbase-cleaned-query', +# 'Database': 'dc_data_baker', +# 'QueryString': "\n UNLOAD (SELECT\n\tsplit_part(postcode, ' ', 1) as outcode,\n\tuprn,\n\taddress,\n\tpostcode,\n\tST_X(ST_GeometryFromText(split_part(location, ';', 2))) as longitude,\n\tST_Y(ST_GeometryFromText(split_part(location, ';', 2))) as latitude,\n\tsubstr(postcode, 1,1) as first_letter\nFROM addressbase_cleaned_raw\n\n) \n TO 's3://pollingstations.private.data/addressbase/testing/addressbase_partitioned/' \n WITH(\n format = 'PARQUET',\n compression = 'SNAPPY',\n partioned_by = ARRAY['first_letter'] \n )\n ", +# 'NamedQueryId': '3a08bb81-4049-4825-a952-aeaf1b94c6f1', +# 'WorkGroup': 'data-baker-workgroup'}, +# 'ResponseMetadata': {'RequestId': '50e5e5bc-a7da-42b7-97d1-e46e7a5f4644', +# 'HTTPStatusCode': 200, +# 'HTTPHeaders': {'date': 'Fri, 26 Apr 2024 06:16:35 GMT', +# 'content-type': 'application/x-amz-json-1.1', +# 'content-length': '780', +# 'connection': 'keep-alive', +# 'x-amzn-requestid': '50e5e5bc-a7da-42b7-97d1-e46e7a5f4644'}, +# 'RetryAttempts': 0}} diff --git a/cdk/lambdas/partition-addressbase-by-first-letter/requirements.in b/cdk/lambdas/partition-addressbase-by-first-letter/requirements.in new file mode 100644 index 0000000..aa5cabe --- /dev/null +++ b/cdk/lambdas/partition-addressbase-by-first-letter/requirements.in @@ -0,0 +1,2 @@ +boto3 +polars diff --git a/cdk/lambdas/partition-addressbase-by-first-letter/requirements.txt b/cdk/lambdas/partition-addressbase-by-first-letter/requirements.txt new file mode 100644 index 0000000..765c38d --- /dev/null +++ b/cdk/lambdas/partition-addressbase-by-first-letter/requirements.txt @@ -0,0 +1,40 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --generate-hashes --output-file cdk/lambdas/partition-addressbase-by-first-letter/requirements.txt cdk/lambdas/partition-addressbase-by-first-letter/requirements.in +boto3==1.34.91 \ + --hash=sha256:5077917041adaaae15eeca340289547ef905ca7e11516e9bd22d394fb5057d2a \ + --hash=sha256:97fac686c47647db4b44e4789317e4aeecd38511d71e84f8d20abe33eb630ff1 +botocore==1.34.91 \ + --hash=sha256:4d1b13f2b1c28ce1743b1e5895ae62bb7e67f892b51882164ea19c27a130852b \ + --hash=sha256:93ef7071292a1b2b9fc26537f8ae3a8227da1177969241939ea3fbdb1a1a1d0c + # via + # boto3 + # s3transfer +jmespath==1.0.1 \ + --hash=sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980 \ + --hash=sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe + # via + # boto3 + # botocore +polars==0.20.22 \ + --hash=sha256:08ee57946f34e2de3ebfc7853d21a14eb92e3993e71d788a6aaaa0757e7bd3e2 \ + --hash=sha256:15d8807828f9c3ddbab60b4aa17ea1dea7743a3dddebfd1c6186826257a687ca \ + --hash=sha256:2f7b08e1725d1a7c522aa316304e8ddb835c69b579577249764c7fa4eeb97305 \ + --hash=sha256:abc5da1f6f7e2ee15bdab74cd19939948a0910799b27ee3eb0768bb95f0e9aff \ + --hash=sha256:ceeb767bb944605539db63c528fe074708f0e23ece2f78f3dfc5132ac2e84d64 \ + --hash=sha256:d211aed6ae34845e1a9766e3b487f73ee9d5044927cc748f7498a72a5a0c8805 +python-dateutil==2.9.0.post0 \ + --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ + --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 + # via botocore +s3transfer==0.10.1 \ + --hash=sha256:5683916b4c724f799e600f41dd9e10a9ff19871bf87623cc8f491cb4f5fa0a19 \ + --hash=sha256:ceb252b11bcf87080fb7850a224fb6e05c8a776bab8f2b64b7f25b969464839d + # via boto3 +six==1.16.0 \ + --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ + --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 + # via python-dateutil +urllib3==2.2.1 \ + --hash=sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d \ + --hash=sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19 + # via botocore diff --git a/cdk/queries/partition-addressbase-cleaned.sql b/cdk/queries/partition-addressbase-cleaned.sql new file mode 100644 index 0000000..c976a7f --- /dev/null +++ b/cdk/queries/partition-addressbase-cleaned.sql @@ -0,0 +1,9 @@ +SELECT + split_part(postcode, ' ', 1) as outcode, + uprn, + address, + postcode, + ST_X(ST_GeometryFromText(split_part(location, ';', 2))) as longitude, + ST_Y(ST_GeometryFromText(split_part(location, ';', 2))) as latitude, + substr(postcode, 1,1) as first_letter +FROM $$from_table$$ diff --git a/cdk/queries/uprn-to-ballots-first-letter.sql b/cdk/queries/uprn-to-ballots-first-letter.sql index d4f312c..d8a5d3b 100644 --- a/cdk/queries/uprn-to-ballots-first-letter.sql +++ b/cdk/queries/uprn-to-ballots-first-letter.sql @@ -1,41 +1,41 @@ -SELECT - combined_results.uprn, - combined_results.address, - combined_results.postcode, - ARRAY_SORT(ARRAY_AGG(distinct combined_results.election_id)) AS ballot_ids, - combined_results.first_letter as first_letter -FROM ( SELECT - ab.uprn, - ab.address, - ab.postcode, - ab.first_letter, - cb.election_id - FROM - current_ballots cb - CROSS JOIN addressbase_partitioned ab - WHERE - ST_CONTAINS( - ST_POLYGON(cb.geometry), - ST_POINT(ab.longitude, ab.latitude) - ) - AND cb.source_table = 'Organisation' - UNION - SELECT - ab.uprn, - ab.address, - ab.postcode, - ab.first_letter, - cb.election_id - FROM - current_ballots cb - CROSS JOIN addressbase_partitioned ab - WHERE - ST_CONTAINS( - ST_Polygon(cb.geometry), - ST_POINT(ab.longitude, ab.latitude) - ) - AND cb.source_table = 'Division' -) AS combined_results -GROUP BY - combined_results.uprn, combined_results.address, combined_results.postcode, combined_results.first_letter + combined_results.uprn, + combined_results.address, + combined_results.postcode, + array_sort(array_agg(distinct combined_results.election_id)) AS ballot_ids, + combined_results.first_letter as first_letter + FROM ( + SELECT + ab.uprn, + ab.address, + ab.postcode, + ab.first_letter, + cb.election_id + FROM + $$from_table$$ cb + CROSS JOIN addressbase_partitioned ab + WHERE + ST_CONTAINS( + ST_Polygon(cb.geometry), + ST_POINT(ab.longitude, ab.latitude) + ) + AND cb.source_table = 'Organisation' + UNION + SELECT + ab.uprn, + ab.address, + ab.postcode, + ab.first_letter, + cb.election_id + FROM + $$from_table$$ cb + CROSS JOIN addressbase_partitioned ab + WHERE + ST_CONTAINS( + ST_Polygon(cb.geometry), + ST_POINT(ab.longitude, ab.latitude) + ) + AND cb.source_table = 'Division' + ) AS combined_results + GROUP BY + combined_results.uprn, combined_results.address, combined_results.postcode, combined_results.first_letter diff --git a/cdk/stacks/data_baker.py b/cdk/stacks/data_baker.py index ef00abc..11cef7c 100644 --- a/cdk/stacks/data_baker.py +++ b/cdk/stacks/data_baker.py @@ -1,20 +1,76 @@ +from pathlib import Path + import aws_cdk.aws_glue_alpha as glue +import aws_cdk.aws_lambda as aws_lambda +import aws_cdk.aws_lambda_python_alpha as aws_lambda_python import aws_cdk.aws_s3 as s3 -from aws_cdk import Stack +from aws_cdk import Duration, Stack +from aws_cdk import aws_athena as athena from constructs import Construct def get_query_text(query): - with open(f"../queries/{query}", "r") as f: + query_path = Path(__file__).parents[1] / "queries" / f"{query}.sql" + with query_path.open("r") as f: return f.read() +class S3Paths: + def __init__(self, dc_environment: str): + self.dc_environment = dc_environment + + assert self.dc_environment in ( + valid_environments := ( + "development", + "staging", + "prod", + ) + ), f"context `dc-environment` must be one of {valid_environments}" + + if self.dc_environment in ( + "development", + "staging", + ): + self.stage = "testing" + else: + self.stage = "current" + + @property + def addressbase_bucket(self) -> str: + return "pollingstations.private.data" + + @property + def ballots_bucket(self) -> str: + return "ee.data-cache.production" + + @property + def addressbase_cleaned_raw_prefix(self) -> str: + return f"addressbase/{self.stage}/addressbase_cleaned/" + + @property + def addressbase_partitioned_prefix(self) -> str: + return f"addressbase/{self.stage}/addressbase_partitioned/" + + @property + def uprn_to_ballots_first_letter(self) -> str: + return f"addressbase/{self.stage}/uprn-to-ballots-first-letter/" + + @property + def current_ballots_prefix(self) -> str: + return "ballots-with-wkt/" + + class DataBakerStack(Stack): def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: super().__init__(scope, construct_id, **kwargs) + # Get DC environment self.dc_environment = self.node.try_get_context("dc-environment") + # Helper for s3 paths + self.s3_paths = S3Paths(self.dc_environment) + + # Glue database self.dc_data_baker_db = glue.Database( self, "dc-data-baker-db", @@ -22,21 +78,67 @@ def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: description="Data base for tables defined by dc data baker", ) - self.pollingstations_private_data_bucket = s3.Bucket.from_bucket_name( + # Buckets + self.addressbase_bucket = s3.Bucket.from_bucket_name( self, - "pollingstations-private-data-bucket", - "pollingstations.private.data", + "addressbase-source-addressbase_bucket", + self.s3_paths.addressbase_bucket, + ) + self.ballots_bucket = s3.Bucket.from_bucket_name( + self, "ballots-source-bucket", self.s3_paths.ballots_bucket + ) + + # Athena workgroup + self.workgroup = self.make_athena_workgroup() + + # Tables + self.addressbase_cleaned_raw_table = ( + self.make_addressbase_cleaned_raw_table() ) + self.addressbase_partitioned_table = ( + self.make_addressbase_partitioned_table() + ) + self.current_ballots_table = self.make_current_ballots_table() + + # Queries + self.define_partition_addressbase_cleaned_query( + self.addressbase_cleaned_raw_table.table_name, + self.addressbase_bucket.s3_url_for_object( + key=self.s3_paths.addressbase_partitioned_prefix + ), + ) + self.define_addressbase_partitioned_msck_repair_query() + self.define_uprn_to_ballots_first_letter_query( + self.current_ballots_table.table_name, + self.addressbase_bucket.s3_url_for_object( + key=self.s3_paths.uprn_to_ballots_first_letter + ), + ) + + # Lambdas + self.make_partition_addressbase_lambda() - self.addressbase_cleaned_raw_table() + def make_partition_addressbase_lambda( + self, + ) -> aws_lambda_python.PythonFunction: + return aws_lambda_python.PythonFunction( + self, + "invoke-partition-addressbase-cleaned-query", + function_name="partition-addressbase-by-first-letter", + entry="cdk/lambdas/partition-addressbase-by-first-letter", + index="handler.py", + handler="handler", + runtime=aws_lambda.Runtime.PYTHON_3_10, + timeout=Duration.minutes(3), + ) - def addressbase_cleaned_raw_table(self): - glue.S3Table( + def make_addressbase_cleaned_raw_table(self) -> glue.S3Table: + return glue.S3Table( self, "addressbase-cleaned-raw-table-id", table_name="addressbase_cleaned_raw", - bucket=self.pollingstations_private_data_bucket, - s3_prefix="addressbase/current/addressbase_cleaned_raw/", + bucket=self.addressbase_bucket, + s3_prefix=self.s3_paths.addressbase_cleaned_raw_prefix, database=self.dc_data_baker_db, columns=[ glue.Column(name="uprn", type=glue.Schema.STRING, comment=""), @@ -62,3 +164,171 @@ def addressbase_cleaned_raw_table(self): data_format=glue.DataFormat.CSV, description="Addressbase table as produced for loading into WDIV", ) + + def make_current_ballots_table(self) -> glue.S3Table: + return glue.S3Table( + self, + "current-ballots-table-id", + table_name="current_ballots", + bucket=self.ballots_bucket, + s3_prefix=self.s3_paths.current_ballots_prefix, + database=self.dc_data_baker_db, + columns=[ + glue.Column( + name="election_id", type=glue.Schema.STRING, comment="" + ), + glue.Column( + name="division_id", type=glue.Schema.STRING, comment="" + ), + glue.Column( + name="geometry", type=glue.Schema.STRING, comment="" + ), + glue.Column( + name="source_table", type=glue.Schema.STRING, comment="" + ), + ], + data_format=glue.DataFormat.CSV, + description="Current ballot ids with wkt as csv", + ) + + def make_addressbase_partitioned_table(self) -> glue.S3Table: + address_partitioned_table = glue.S3Table( + self, + "addressbase-partitioned-table-id", + table_name="addressbase_partitioned", + bucket=self.addressbase_bucket, + s3_prefix=self.s3_paths.addressbase_partitioned_prefix, + database=self.dc_data_baker_db, + storage_parameters=[ + glue.StorageParameter.custom("serialization.format", "1") + ], + columns=[ + glue.Column( + name="outcode", + type=glue.Schema.STRING, + ), + glue.Column( + name="uprn", + type=glue.Schema.STRING, + ), + glue.Column( + name="address", + type=glue.Schema.STRING, + ), + glue.Column( + name="postcode", + type=glue.Schema.STRING, + ), + glue.Column( + name="longitude", + type=glue.Schema.DOUBLE, + ), + glue.Column( + name="latitude", + type=glue.Schema.DOUBLE, + ), + ], + partition_keys=[ + glue.Column( + name="first_letter", + type=glue.Schema.STRING, + ) + ], + data_format=glue.DataFormat.PARQUET, + description="Addressbase table partitioned by first letter of postcode. " + "With latitude and longitude columns", + ) + address_partitioned_table.add_partition_index( + key_names=["first_letter"], index_name="first_letter" + ) + return address_partitioned_table + + def partition_addressbase_cleaned_query_string( + self, from_table: str, s3_target_dir: str + ) -> str: + base_query = get_query_text("partition-addressbase-cleaned") + base_query = base_query.replace("$$from_table$$", from_table) + return f""" + UNLOAD ({base_query}) + TO '{s3_target_dir}' + WITH( + format = 'PARQUET', + compression = 'SNAPPY', + partitioned_by = ARRAY['first_letter'] + ) + """ + + def uprn_to_ballots_first_letter_query_string( + self, from_table: str, s3_target_dir: str + ) -> str: + base_query = get_query_text("uprn-to-ballots-first-letter") + base_query = base_query.replace("$$from_table$$", from_table) + return f""" + UNLOAD ({base_query}) + TO '{s3_target_dir}' + WITH( + format = 'PARQUET', + compression = 'SNAPPY', + partitioned_by = ARRAY['first_letter'] + ) + """ + + def define_partition_addressbase_cleaned_query( + self, from_table: str, s3_target_dir: str + ) -> athena.CfnNamedQuery: + name = "partition-addressbase-cleaned" + return athena.CfnNamedQuery( + self, + f"{name}-id", + database=self.dc_data_baker_db.database_name, + query_string=self.partition_addressbase_cleaned_query_string( + from_table, s3_target_dir + ), + name=name, + work_group=self.workgroup.name, + ) + + def define_addressbase_partitioned_msck_repair_query( + self, + ) -> athena.CfnNamedQuery: + name = "addressbase-partitioned-msck-repair" + return athena.CfnNamedQuery( + self, + f"{name}-id", + database=self.dc_data_baker_db.database_name, + query_string=f"MSCK REPAIR TABLE {self.addressbase_partitioned_table.table_name}", + name=name, + work_group=self.workgroup.name, + ) + + def define_uprn_to_ballots_first_letter_query( + self, from_table: str, s3_target_dir: str + ) -> athena.CfnNamedQuery: + name = "uprn-to-ballots-first-letter" + return athena.CfnNamedQuery( + self, + f"{name}-id", + database=self.dc_data_baker_db.database_name, + query_string=self.uprn_to_ballots_first_letter_query_string( + from_table, s3_target_dir + ), + name=name, + work_group=self.workgroup.name, + ) + + def make_athena_workgroup(self) -> athena.CfnWorkGroup: + return athena.CfnWorkGroup( + self, + "dc-data-baker-workgroup-id", + name="dc-data-baker", + work_group_configuration=athena.CfnWorkGroup.WorkGroupConfigurationProperty( + result_configuration=athena.CfnWorkGroup.ResultConfigurationProperty( + output_location=self.results_bucket().s3_url_for_object( + key="dc-data-baker-athena-results" + ) + ), + ), + ) + + def results_bucket(self) -> s3.Bucket: + return s3.Bucket(self, "dc-data-baker-results-addressbase-bucket") diff --git a/requirements/base.in b/requirements/base.in index 4aeae27..33c78cc 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -1,3 +1,4 @@ aws-cdk-lib >= 2.0.0, < 3.0.0 aws-cdk.aws-glue-alpha >= 2.0.0, < 3.0.0 +aws-cdk.aws_lambda_python_alpha >= 2.0.0, < 3.0.0 constructs >= 10.0.0 diff --git a/requirements/base.txt b/requirements/base.txt index 07d5705..6c8e1f5 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -21,10 +21,15 @@ aws-cdk-asset-node-proxy-agent-v6==2.0.3 \ aws-cdk-aws-glue-alpha==2.138.0a0 \ --hash=sha256:505bd0bda04c061f390dcd1c75a28fc827e784e9a83ed7c9337015c034b37b59 \ --hash=sha256:58c3b4f467554334acd635a37bc8562e448ce8bdaa9f0879d5b0a547f4fe1e9e +aws-cdk-aws-lambda-python-alpha==2.138.0a0 \ + --hash=sha256:70626e0a3ac992dd44111b4f99584e2fec7d71c7d7c270ab20c85a27c42afcd1 \ + --hash=sha256:7f76bb9eafb7286d10dcf3876814a4cb10abc856e14710740af680f9f8128bbf aws-cdk-lib==2.138.0 \ --hash=sha256:a3f69d1ea22271db4a89a28f3ccf6995c6591586aee0447766933488662a36d1 \ --hash=sha256:b528fe909dc0e8686f086369fe38bde6f448efbe9519dc56aef3be35d3da75f7 - # via aws-cdk-aws-glue-alpha + # via + # aws-cdk-aws-glue-alpha + # aws-cdk-aws-lambda-python-alpha cattrs==23.2.3 \ --hash=sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108 \ --hash=sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f @@ -34,7 +39,12 @@ constructs==10.3.0 \ --hash=sha256:518551135ec236f9cc6b86500f4fbbe83b803ccdc6c2cb7684e0b7c4d234e7b1 # via # aws-cdk-aws-glue-alpha + # aws-cdk-aws-lambda-python-alpha # aws-cdk-lib +exceptiongroup==1.2.1 \ + --hash=sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad \ + --hash=sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16 + # via cattrs importlib-resources==6.4.0 \ --hash=sha256:50d10f043df931902d4194ea07ec57960f66a80449ff867bfe782b4c486ba78c \ --hash=sha256:cdb2b453b8046ca4e3798eb1d84f3cce1446a0e8e7b5ef4efb600f19fc398145 @@ -47,6 +57,7 @@ jsii==1.97.0 \ # aws-cdk-asset-kubectl-v20 # aws-cdk-asset-node-proxy-agent-v6 # aws-cdk-aws-glue-alpha + # aws-cdk-aws-lambda-python-alpha # aws-cdk-lib # constructs publication==0.0.3 \ @@ -57,6 +68,7 @@ publication==0.0.3 \ # aws-cdk-asset-kubectl-v20 # aws-cdk-asset-node-proxy-agent-v6 # aws-cdk-aws-glue-alpha + # aws-cdk-aws-lambda-python-alpha # aws-cdk-lib # constructs # jsii @@ -76,10 +88,13 @@ typeguard==2.13.3 \ # aws-cdk-asset-kubectl-v20 # aws-cdk-asset-node-proxy-agent-v6 # aws-cdk-aws-glue-alpha + # aws-cdk-aws-lambda-python-alpha # aws-cdk-lib # constructs # jsii typing-extensions==4.11.0 \ --hash=sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0 \ --hash=sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a - # via jsii + # via + # cattrs + # jsii diff --git a/requirements/dev.in b/requirements/dev.in index 9883e72..72be42d 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -1,2 +1,5 @@ +-c base.txt + +ipython ruff -pre-commit \ No newline at end of file +pre-commit diff --git a/requirements/dev.txt b/requirements/dev.txt index 3c87c4e..f25e102 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,42 +1,29 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --generate-hashes --output-file requirements/dev.txt requirements/dev.in requirements/base.in -attrs==23.2.0 \ - --hash=sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30 \ - --hash=sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1 - # via - # cattrs - # jsii -aws-cdk-asset-awscli-v1==2.2.202 \ - --hash=sha256:3ef87d6530736b3a7b0f777fe3b4297994dd40c3ce9306d95f80f48fb18036e8 \ - --hash=sha256:96205ea2e5e132ec52fabfff37ea25b9b859498f167d05b32564c949822cd331 - # via aws-cdk-lib -aws-cdk-asset-kubectl-v20==2.1.2 \ - --hash=sha256:346283e43018a43e3b3ca571de3f44e85d49c038dc20851894cb8f9b2052b164 \ - --hash=sha256:7f0617ab6cb942b066bd7174bf3e1f377e57878c3e1cddc21d6b2d13c92d0cc1 - # via aws-cdk-lib -aws-cdk-asset-node-proxy-agent-v6==2.0.3 \ - --hash=sha256:b62cb10c69a42cab135e6bc670e3d2d3121fd4f53a0f61e53449da4b12738a6f \ - --hash=sha256:ef2ff0634ab037e2ebddbe69d7c92515a847c6c8bb2abdfc85b089f5e87761cb - # via aws-cdk-lib -aws-cdk-lib==2.138.0 \ - --hash=sha256:a3f69d1ea22271db4a89a28f3ccf6995c6591586aee0447766933488662a36d1 \ - --hash=sha256:b528fe909dc0e8686f086369fe38bde6f448efbe9519dc56aef3be35d3da75f7 -cattrs==23.2.3 \ - --hash=sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108 \ - --hash=sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f - # via jsii +# uv pip compile --generate-hashes --output-file requirements/dev.txt requirements/dev.in +asttokens==2.4.1 \ + --hash=sha256:051ed49c3dcae8913ea7cd08e46a606dba30b79993209636c4875bc1d637bc24 \ + --hash=sha256:b03869718ba9a6eb027e134bfdf69f38a236d681c83c160d510768af11254ba0 + # via stack-data cfgv==3.4.0 \ --hash=sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9 \ --hash=sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560 # via pre-commit -constructs==10.3.0 \ - --hash=sha256:2972f514837565ff5b09171cfba50c0159dfa75ee86a42921ea8c86f2941b3d2 \ - --hash=sha256:518551135ec236f9cc6b86500f4fbbe83b803ccdc6c2cb7684e0b7c4d234e7b1 - # via aws-cdk-lib +decorator==5.1.1 \ + --hash=sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330 \ + --hash=sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186 + # via ipython distlib==0.3.8 \ --hash=sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784 \ --hash=sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64 # via virtualenv +exceptiongroup==1.2.1 \ + --hash=sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad \ + --hash=sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16 + # via ipython +executing==2.0.1 \ + --hash=sha256:35afe2ce3affba8ee97f2d69927fa823b08b472b7b994e36a52a964b93d16147 \ + --hash=sha256:eac49ca94516ccc753f9fb5ce82603156e590b27525a8bc32cce8ae302eb61bc + # via stack-data filelock==3.13.4 \ --hash=sha256:404e5e9253aa60ad457cae1be07c0f0ca90a63931200a47d9b6a6af84fd7b45f \ --hash=sha256:d13f466618bfde72bd2c18255e269f72542c6e70e7bac83a0232d6b1cc5c8cf4 @@ -45,23 +32,29 @@ identify==2.5.36 \ --hash=sha256:37d93f380f4de590500d9dba7db359d0d3da95ffe7f9de1753faa159e71e7dfa \ --hash=sha256:e5e00f54165f9047fbebeb4a560f9acfb8af4c88232be60a488e9b68d122745d # via pre-commit -importlib-resources==6.4.0 \ - --hash=sha256:50d10f043df931902d4194ea07ec57960f66a80449ff867bfe782b4c486ba78c \ - --hash=sha256:cdb2b453b8046ca4e3798eb1d84f3cce1446a0e8e7b5ef4efb600f19fc398145 - # via jsii -jsii==1.97.0 \ - --hash=sha256:5dd347cc9d279072c109829aaff11dae1c7f13169ce60887f1c1ab2c4cd4abcd \ - --hash=sha256:e6db98e34730cd972d180b7f4e21182b9a5105f537672716940b930ee933a1f2 - # via - # aws-cdk-asset-awscli-v1 - # aws-cdk-asset-kubectl-v20 - # aws-cdk-asset-node-proxy-agent-v6 - # aws-cdk-lib - # constructs +ipython==8.23.0 \ + --hash=sha256:07232af52a5ba146dc3372c7bf52a0f890a23edf38d77caef8d53f9cdc2584c1 \ + --hash=sha256:7468edaf4f6de3e1b912e57f66c241e6fd3c7099f2ec2136e239e142e800274d +jedi==0.19.1 \ + --hash=sha256:cf0496f3651bc65d7174ac1b7d043eff454892c708a87d1b683e57b569927ffd \ + --hash=sha256:e983c654fe5c02867aef4cdfce5a2fbb4a50adc0af145f70504238f18ef5e7e0 + # via ipython +matplotlib-inline==0.1.7 \ + --hash=sha256:8423b23ec666be3d16e16b60bdd8ac4e86e840ebd1dd11a30b9f117f2fa0ab90 \ + --hash=sha256:df192d39a4ff8f21b1895d72e6a13f5fcc5099f00fa84384e0ea28c2cc0653ca + # via ipython nodeenv==1.8.0 \ --hash=sha256:d51e0c37e64fbf47d017feac3145cdbb58836d7eee8c6f6d3b6880c5456227d2 \ --hash=sha256:df865724bb3c3adc86b3876fa209771517b0cfe596beff01a92700e0e8be4cec # via pre-commit +parso==0.8.4 \ + --hash=sha256:a418670a20291dacd2dddc80c377c5c3791378ee1e8d12bffc35420643d43f18 \ + --hash=sha256:eb3a7b58240fb99099a345571deecc0f9540ea5f4dd2fe14c2a99d6b281ab92d + # via jedi +pexpect==4.9.0 \ + --hash=sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523 \ + --hash=sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f + # via ipython platformdirs==4.2.0 \ --hash=sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068 \ --hash=sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768 @@ -69,20 +62,22 @@ platformdirs==4.2.0 \ pre-commit==3.7.0 \ --hash=sha256:5eae9e10c2b5ac51577c3452ec0a490455c45a0533f7960f993a0d01e59decab \ --hash=sha256:e209d61b8acdcf742404408531f0c37d49d2c734fd7cff2d6076083d191cb060 -publication==0.0.3 \ - --hash=sha256:0248885351febc11d8a1098d5c8e3ab2dabcf3e8c0c96db1e17ecd12b53afbe6 \ - --hash=sha256:68416a0de76dddcdd2930d1c8ef853a743cc96c82416c4e4d3b5d901c6276dc4 - # via - # aws-cdk-asset-awscli-v1 - # aws-cdk-asset-kubectl-v20 - # aws-cdk-asset-node-proxy-agent-v6 - # aws-cdk-lib - # constructs - # jsii -python-dateutil==2.9.0.post0 \ - --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ - --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 - # via jsii +prompt-toolkit==3.0.43 \ + --hash=sha256:3527b7af26106cbc65a040bcc84839a3566ec1b051bb0bfe953631e704b0ff7d \ + --hash=sha256:a11a29cb3bf0a28a387fe5122cdb649816a957cd9261dcedf8c9f1fef33eacf6 + # via ipython +ptyprocess==0.7.0 \ + --hash=sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35 \ + --hash=sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220 + # via pexpect +pure-eval==0.2.2 \ + --hash=sha256:01eaab343580944bc56080ebe0a674b39ec44a945e6d09ba7db3cb8cec289350 \ + --hash=sha256:2b45320af6dfaa1750f543d714b6d1c520a1688dec6fd24d339063ce0aaa9ac3 + # via stack-data +pygments==2.17.2 \ + --hash=sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c \ + --hash=sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367 + # via ipython pyyaml==6.0.1 \ --hash=sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5 \ --hash=sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc \ @@ -161,22 +156,26 @@ setuptools==69.5.1 \ six==1.16.0 \ --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 - # via python-dateutil -typeguard==2.13.3 \ - --hash=sha256:00edaa8da3a133674796cf5ea87d9f4b4c367d77476e185e80251cc13dfbb8c4 \ - --hash=sha256:5e3e3be01e887e7eafae5af63d1f36c849aaa94e3a0112097312aabfa16284f1 + # via asttokens +stack-data==0.6.3 \ + --hash=sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9 \ + --hash=sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695 + # via ipython +traitlets==5.14.3 \ + --hash=sha256:9ed0579d3502c94b4b3732ac120375cda96f923114522847de4b3bb98b96b6b7 \ + --hash=sha256:b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f # via - # aws-cdk-asset-awscli-v1 - # aws-cdk-asset-kubectl-v20 - # aws-cdk-asset-node-proxy-agent-v6 - # aws-cdk-lib - # constructs - # jsii + # ipython + # matplotlib-inline typing-extensions==4.11.0 \ --hash=sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0 \ --hash=sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a - # via jsii + # via ipython virtualenv==20.25.3 \ --hash=sha256:7bb554bbdfeaacc3349fa614ea5bff6ac300fc7c335e9facf3a3bcfc703f45be \ --hash=sha256:8aac4332f2ea6ef519c648d0bc48a5b1d324994753519919bddbb1aff25a104e # via pre-commit +wcwidth==0.2.13 \ + --hash=sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859 \ + --hash=sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5 + # via prompt-toolkit