From 3a64da564a2cc9c85fa84840d97d645e335a0008 Mon Sep 17 00:00:00 2001 From: Bo Xu Date: Thu, 5 Oct 2023 17:06:01 +0000 Subject: [PATCH] Bundle import tool into RSI docker image; Add website endpoint for RSI data load (#3667) --- .gitignore | 1 + .gitmodules | 5 +- build/python_node_go_protoc_envoy/Dockerfile | 2 +- .../cloudbuild.yaml | 2 +- build/web_compose/Dockerfile | 9 +- docs/custom_dc.md | 10 +- import | 1 + mixer | 2 +- server/config/subject_page_pb2.py | 40 +-- .../chart_config.json | 2 +- .../translate_hindi/query_1/chart_config.json | 324 ++++++++++++++++++ server/routes/import_wizard/html.py | 61 +++- 12 files changed, 426 insertions(+), 33 deletions(-) create mode 160000 import diff --git a/.gitignore b/.gitignore index e3b8f56605..ed6880c4cc 100644 --- a/.gitignore +++ b/.gitignore @@ -66,6 +66,7 @@ experimental/sdg-static/datacommons/nl_interface.min.css # Custom DC data data/ +dc-data/ # Topic cache gen_ordered_list_for_topics.mcf diff --git a/.gitmodules b/.gitmodules index 6aeca29911..91dffd7f82 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "mixer"] path = mixer - url = https://github.com/datacommonsorg/mixer.git \ No newline at end of file + url = https://github.com/datacommonsorg/mixer.git +[submodule "import"] + path = import + url = https://github.com/datacommonsorg/import.git diff --git a/build/python_node_go_protoc_envoy/Dockerfile b/build/python_node_go_protoc_envoy/Dockerfile index ca5f100fc8..7d98be3a0f 100644 --- a/build/python_node_go_protoc_envoy/Dockerfile +++ b/build/python_node_go_protoc_envoy/Dockerfile @@ -16,7 +16,7 @@ FROM python:3.11.4-slim as base RUN apt update -RUN apt install -y curl zip wget +RUN apt install -y curl zip wget gcc libc6-dev # Install Node ENV NODE_VERSION=18.17.1 diff --git a/build/python_node_go_protoc_envoy/cloudbuild.yaml b/build/python_node_go_protoc_envoy/cloudbuild.yaml index e906e6aec8..acbce7a464 100644 --- a/build/python_node_go_protoc_envoy/cloudbuild.yaml +++ b/build/python_node_go_protoc_envoy/cloudbuild.yaml @@ -13,7 +13,7 @@ # limitations under the License. substitutions: - _VERS: "2023-08-19" + _VERS: "2023-10-05" steps: - name: "gcr.io/cloud-builders/docker" diff --git a/build/web_compose/Dockerfile b/build/web_compose/Dockerfile index 5fba56d98f..25f2c9ff0c 100644 --- a/build/web_compose/Dockerfile +++ b/build/web_compose/Dockerfile @@ -13,7 +13,7 @@ # limitations under the License. -FROM gcr.io/datcom-ci/python-node-go-protoc-envoy:2023-08-19 +FROM gcr.io/datcom-ci/python-node-go-protoc-envoy:2023-10-05 ARG ENV ENV ENV=${ENV} @@ -48,6 +48,7 @@ RUN GRPC_HEALTH_PROBE_VERSION=v0.4.7 && \ # Build binary COPY mixer/cmd/ cmd COPY mixer/esp/ esp +ENV CGO_ENABLED=1 RUN go build -o /go/bin/mixer cmd/main.go # Flask @@ -57,6 +58,8 @@ RUN pip install --upgrade pip RUN pip install --upgrade setuptools # --no-cache-dir removes ~/.cache files, which can be a few GB. RUN pip3 install --no-cache-dir -r /workspace/server/requirements.txt +# Used by Import Tool +RUN pip install pandas # NPM WORKDIR /workspace/static @@ -72,6 +75,10 @@ RUN npm run-script build COPY server/. /workspace/server/ COPY shared/. /workspace/shared/ +# Import Tool +COPY import/. /workspace/import/ + +# Env ENV USE_SQLITE=false ENV USE_CLOUDSQL=false ENV REMOTE_MIXER_DOMAIN=https://api.datacommons.org diff --git a/docs/custom_dc.md b/docs/custom_dc.md index f5162a750b..536e4d4b22 100644 --- a/docs/custom_dc.md +++ b/docs/custom_dc.md @@ -82,11 +82,14 @@ docker run -it --pull=always \ -e USE_SQLITE=true \ -e SQL_DATA_PATH=/sqlite \ -p 8080:8080 \ --p 8081:8081 \ -v $HOME/dc-data:/sqlite \ gcr.io/datcom-ci/datacommons-website-compose:latest ``` +Run `curl -X POST localhost:8080/import/simple/load2sql` to load the data into +database. Whenever there is update on the data, re-run this commands (no need to +restart the container). + If you have your own UI updates and build the docker image locally, replace the docker image with your locally one in the command. @@ -94,10 +97,6 @@ Now you can access a custom Data Commons site via [localhost](http://localhost:8080). For example, the data from the sample data can be viewed in [Timeline Chart](http://localhost:8080/tools/timeline#place=geoId%2F06&statsVar=stat_var_1). -If the CSV files are updated, just run `curl -X POST localhost:8081/import`. -This re-imports the CSV files into the local instance without the need to -restart the container. - ## Run in Cloud Custom Data Commons can be ran on the cloud as a production service. The SQLite @@ -148,7 +147,6 @@ docker run -it \ -e GOOGLE_APPLICATION_CREDENTIALS=/gcp/creds.json \ -v $HOME/.config/gcloud/application_default_credentials.json:/gcp/creds.json:ro \ -p 8080:8080 \ --p 8081:8081 \ gcr.io/datcom-ci/datacommons-website-compose:latest ``` diff --git a/import b/import new file mode 160000 index 0000000000..fd9e74fcfa --- /dev/null +++ b/import @@ -0,0 +1 @@ +Subproject commit fd9e74fcfaf928cc46f1e976fde3f916f81194f8 diff --git a/mixer b/mixer index 8aab9f1ee8..d268ecf465 160000 --- a/mixer +++ b/mixer @@ -1 +1 @@ -Subproject commit 8aab9f1ee839ef7df9f0bb48ea859c8297185b71 +Subproject commit d268ecf46500248e5efd2e9c8ee8403437df5d91 diff --git a/server/config/subject_page_pb2.py b/server/config/subject_page_pb2.py index b60486693a..9b107aac2d 100644 --- a/server/config/subject_page_pb2.py +++ b/server/config/subject_page_pb2.py @@ -13,7 +13,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x12subject_page.proto\x12\x0b\x64\x61tacommons\"l\n\x0eSeverityFilter\x12\x0c\n\x04prop\x18\x01 \x01(\t\x12\x14\n\x0c\x64isplay_name\x18\x05 \x01(\t\x12\x0c\n\x04unit\x18\x02 \x01(\t\x12\x13\n\x0blower_limit\x18\x03 \x01(\x01\x12\x13\n\x0bupper_limit\x18\x04 \x01(\x01\"\x9b\x04\n\rEventTypeSpec\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x18\n\x10\x65vent_type_dcids\x18\x03 \x03(\t\x12\r\n\x05\x63olor\x18\x04 \x01(\t\x12<\n\x17\x64\x65\x66\x61ult_severity_filter\x18\x05 \x01(\x0b\x32\x1b.datacommons.SeverityFilter\x12[\n\x1aplace_type_severity_filter\x18\n \x03(\x0b\x32\x37.datacommons.EventTypeSpec.PlaceTypeSeverityFilterEntry\x12<\n\x0c\x64isplay_prop\x18\x06 \x03(\x0b\x32&.datacommons.EventTypeSpec.DisplayProp\x12\x15\n\rend_date_prop\x18\x07 \x03(\t\x12\x1d\n\x15polygon_geo_json_prop\x18\x08 \x01(\t\x12\x1a\n\x12path_geo_json_prop\x18\t \x01(\t\x1a[\n\x1cPlaceTypeSeverityFilterEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12*\n\x05value\x18\x02 \x01(\x0b\x32\x1b.datacommons.SeverityFilter:\x02\x38\x01\x1a?\n\x0b\x44isplayProp\x12\x0c\n\x04prop\x18\x01 \x01(\t\x12\x14\n\x0c\x64isplay_name\x18\x02 \x01(\t\x12\x0c\n\x04unit\x18\x03 \x01(\t\"\xe3\x03\n\x0cPageMetadata\x12\x10\n\x08topic_id\x18\x01 \x01(\t\x12\x12\n\ntopic_name\x18\x02 \x01(\t\x12\x12\n\nplace_dcid\x18\x03 \x03(\t\x12Q\n\x15\x63ontained_place_types\x18\x04 \x03(\x0b\x32\x32.datacommons.PageMetadata.ContainedPlaceTypesEntry\x12\x45\n\x0f\x65vent_type_spec\x18\x05 \x03(\x0b\x32,.datacommons.PageMetadata.EventTypeSpecEntry\x12\x39\n\x0bplace_group\x18\x06 \x03(\x0b\x32$.datacommons.PageMetadata.PlaceGroup\x1a:\n\x18\x43ontainedPlaceTypesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1aP\n\x12\x45ventTypeSpecEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12)\n\x05value\x18\x02 \x01(\x0b\x32\x1a.datacommons.EventTypeSpec:\x02\x38\x01\x1a\x36\n\nPlaceGroup\x12\x14\n\x0cparent_place\x18\x01 \x01(\t\x12\x12\n\nplace_type\x18\x02 \x01(\t\"\x9f\x01\n\x0bStatVarSpec\x12\x10\n\x08stat_var\x18\x01 \x01(\t\x12\r\n\x05\x64\x65nom\x18\x02 \x01(\t\x12\x0c\n\x04unit\x18\x03 \x01(\t\x12\x0f\n\x07scaling\x18\x04 \x01(\x01\x12\x0b\n\x03log\x18\x05 \x01(\x08\x12\x0c\n\x04name\x18\x06 \x01(\t\x12\x0c\n\x04\x64\x61te\x18\x07 \x01(\t\x12\x15\n\rno_per_capita\x18\x08 \x01(\x08\x12\x10\n\x08\x66\x61\x63\x65t_id\x18\t \x01(\t\"\xd0\x01\n\x0fRankingTileSpec\x12\x14\n\x0cshow_highest\x18\x01 \x01(\x08\x12\x13\n\x0bshow_lowest\x18\x02 \x01(\x08\x12\x16\n\x0e\x64iff_base_date\x18\x05 \x01(\t\x12\x15\n\rhighest_title\x18\x06 \x01(\t\x12\x14\n\x0clowest_title\x18\x07 \x01(\t\x12\x15\n\rranking_count\x18\n \x01(\x05\x12\x19\n\x11show_multi_column\x18\x0b \x01(\x08\x12\x1b\n\x13show_highest_lowest\x18\x0c \x01(\x08\"u\n\x18\x44isasterEventMapTileSpec\x12\x1c\n\x14point_event_type_key\x18\x01 \x03(\t\x12\x1e\n\x16polygon_event_type_key\x18\x02 \x03(\t\x12\x1b\n\x13path_event_type_key\x18\x03 \x03(\t\"9\n\x11HistogramTileSpec\x12\x16\n\x0e\x65vent_type_key\x18\x01 \x01(\t\x12\x0c\n\x04prop\x18\x02 \x01(\t\"\x9d\x01\n\x10TopEventTileSpec\x12\x16\n\x0e\x65vent_type_key\x18\x01 \x01(\t\x12\x14\n\x0c\x64isplay_prop\x18\x02 \x03(\t\x12\x17\n\x0fshow_start_date\x18\x03 \x01(\x08\x12\x15\n\rshow_end_date\x18\x04 \x01(\x08\x12\x14\n\x0creverse_sort\x18\x05 \x01(\x08\x12\x15\n\rranking_count\x18\x06 \x01(\x05\"\xbc\x01\n\x0fScatterTileSpec\x12\x1b\n\x13highlight_top_right\x18\x01 \x01(\x08\x12\x1a\n\x12highlight_top_left\x18\x02 \x01(\x08\x12\x1e\n\x16highlight_bottom_right\x18\x03 \x01(\x08\x12\x1d\n\x15highlight_bottom_left\x18\x04 \x01(\x08\x12\x19\n\x11show_place_labels\x18\x05 \x01(\x08\x12\x16\n\x0eshow_quadrants\x18\x06 \x01(\x08\"\xf0\x02\n\x0b\x42\x61rTileSpec\x12\x19\n\x11x_label_link_root\x18\x01 \x01(\t\x12\x12\n\nbar_height\x18\x02 \x01(\x01\x12\x0e\n\x06\x63olors\x18\x03 \x03(\t\x12\x12\n\nhorizontal\x18\x04 \x01(\x08\x12\x12\n\nmax_places\x18\x05 \x01(\x05\x12\x15\n\rmax_variables\x18\x06 \x01(\x05\x12/\n\x04sort\x18\x07 \x01(\x0e\x32!.datacommons.BarTileSpec.SortType\x12\x0f\n\x07stacked\x18\x08 \x01(\x08\x12\x14\n\x0cuse_lollipop\x18\t \x01(\x08\x12\x15\n\ry_axis_margin\x18\n \x01(\x01\"t\n\x08SortType\x12\x14\n\x10TYPE_UNSPECIFIED\x10\x00\x12\r\n\tASCENDING\x10\x01\x12\x0e\n\nDESCENDING\x10\x02\x12\x18\n\x14\x41SCENDING_POPULATION\x10\x03\x12\x19\n\x15\x44\x45SCENDING_POPULATION\x10\x04\"s\n\rGaugeTileSpec\x12/\n\x05range\x18\x01 \x01(\x0b\x32 .datacommons.GaugeTileSpec.Range\x12\x0e\n\x06\x63olors\x18\x02 \x03(\t\x1a!\n\x05Range\x12\x0b\n\x03min\x18\x01 \x01(\x01\x12\x0b\n\x03max\x18\x02 \x01(\x01\",\n\rDonutTileSpec\x12\x0e\n\x06\x63olors\x18\x01 \x03(\t\x12\x0b\n\x03pie\x18\x02 \x01(\x08\"\x1e\n\x0cLineTileSpec\x12\x0e\n\x06\x63olors\x18\x01 \x03(\t\"\x1d\n\x0bMapTileSpec\x12\x0e\n\x06\x63olors\x18\x02 \x03(\t\"\xe0\x07\n\x04Tile\x12\r\n\x05title\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12(\n\x04type\x18\x03 \x01(\x0e\x32\x1a.datacommons.Tile.TileType\x12\x14\n\x0cstat_var_key\x18\x04 \x03(\t\x12\x19\n\x11\x63omparison_places\x18\x07 \x03(\t\x12\x1b\n\x13place_dcid_override\x18\x0b \x01(\t\x12\x39\n\x11ranking_tile_spec\x18\x05 \x01(\x0b\x32\x1c.datacommons.RankingTileSpecH\x00\x12M\n\x1c\x64isaster_event_map_tile_spec\x18\x06 \x01(\x0b\x32%.datacommons.DisasterEventMapTileSpecH\x00\x12<\n\x13top_event_tile_spec\x18\x08 \x01(\x0b\x32\x1d.datacommons.TopEventTileSpecH\x00\x12\x39\n\x11scatter_tile_spec\x18\t \x01(\x0b\x32\x1c.datacommons.ScatterTileSpecH\x00\x12=\n\x13histogram_tile_spec\x18\n \x01(\x0b\x32\x1e.datacommons.HistogramTileSpecH\x00\x12\x31\n\rbar_tile_spec\x18\x0c \x01(\x0b\x32\x18.datacommons.BarTileSpecH\x00\x12\x35\n\x0fgauge_tile_spec\x18\r \x01(\x0b\x32\x1a.datacommons.GaugeTileSpecH\x00\x12\x35\n\x0f\x64onut_tile_spec\x18\x0e \x01(\x0b\x32\x1a.datacommons.DonutTileSpecH\x00\x12\x33\n\x0eline_tile_spec\x18\x0f \x01(\x0b\x32\x19.datacommons.LineTileSpecH\x00\x12\x31\n\rmap_tile_spec\x18\x10 \x01(\x0b\x32\x18.datacommons.MapTileSpecH\x00\"\xde\x01\n\x08TileType\x12\r\n\tTYPE_NONE\x10\x00\x12\x08\n\x04LINE\x10\x01\x12\x07\n\x03\x42\x41R\x10\x02\x12\x07\n\x03MAP\x10\x03\x12\x0b\n\x07SCATTER\x10\x04\x12\r\n\tBIVARIATE\x10\x05\x12\x0b\n\x07RANKING\x10\x06\x12\r\n\tHIGHLIGHT\x10\x07\x12\x0f\n\x0b\x44\x45SCRIPTION\x10\x08\x12\t\n\x05GAUGE\x10\r\x12\t\n\x05\x44ONUT\x10\x0e\x12\r\n\tHISTOGRAM\x10\n\x12\x12\n\x0ePLACE_OVERVIEW\x10\x0b\x12\r\n\tTOP_EVENT\x10\x0c\x12\x16\n\x12\x44ISASTER_EVENT_MAP\x10\tB\x10\n\x0etile_type_spec\"\xcf\x01\n\x11\x44isasterBlockSpec\x12>\n\ndate_range\x18\x01 \x01(\x0e\x32(.datacommons.DisasterBlockSpec.DateRangeH\x00\x12\x0e\n\x04\x64\x61te\x18\x02 \x01(\tH\x00\"Z\n\tDateRange\x12\r\n\tTYPE_NONE\x10\x00\x12\x0f\n\x0bTHIRTY_DAYS\x10\x01\x12\x0e\n\nSIX_MONTHS\x10\x02\x12\x0c\n\x08ONE_YEAR\x10\x03\x12\x0f\n\x0bTHREE_YEARS\x10\x04\x42\x0e\n\x0c\x64\x65\x66\x61ult_date\"\xec\x02\n\x05\x42lock\x12\r\n\x05title\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x10\n\x08\x66ootnote\x18\x05 \x01(\t\x12*\n\x07\x63olumns\x18\x03 \x03(\x0b\x32\x19.datacommons.Block.Column\x12*\n\x04type\x18\x04 \x01(\x0e\x32\x1c.datacommons.Block.BlockType\x12\r\n\x05\x64\x65nom\x18\x06 \x01(\t\x12\x18\n\x10start_with_denom\x18\x07 \x01(\x08\x12=\n\x13\x64isaster_block_spec\x18\x08 \x01(\x0b\x32\x1e.datacommons.DisasterBlockSpecH\x00\x1a*\n\x06\x43olumn\x12 \n\x05tiles\x18\x01 \x03(\x0b\x32\x11.datacommons.Tile\".\n\tBlockType\x12\r\n\tTYPE_NONE\x10\x00\x12\x12\n\x0e\x44ISASTER_EVENT\x10\x01\x42\x11\n\x0f\x62lock_type_spec\"\xed\x01\n\x08\x43\x61tegory\x12\r\n\x05title\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12=\n\rstat_var_spec\x18\x04 \x03(\x0b\x32&.datacommons.Category.StatVarSpecEntry\x12\"\n\x06\x62locks\x18\x03 \x03(\x0b\x32\x12.datacommons.Block\x12\x0c\n\x04\x64\x63id\x18\x05 \x01(\t\x1aL\n\x10StatVarSpecEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\'\n\x05value\x18\x02 \x01(\x0b\x32\x18.datacommons.StatVarSpec:\x02\x38\x01\"q\n\x11SubjectPageConfig\x12+\n\x08metadata\x18\x01 \x01(\x0b\x32\x19.datacommons.PageMetadata\x12)\n\ncategories\x18\x02 \x03(\x0b\x32\x15.datacommons.CategoryJ\x04\x08\x03\x10\x04\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x12subject_page.proto\x12\x0b\x64\x61tacommons\"l\n\x0eSeverityFilter\x12\x0c\n\x04prop\x18\x01 \x01(\t\x12\x14\n\x0c\x64isplay_name\x18\x05 \x01(\t\x12\x0c\n\x04unit\x18\x02 \x01(\t\x12\x13\n\x0blower_limit\x18\x03 \x01(\x01\x12\x13\n\x0bupper_limit\x18\x04 \x01(\x01\"\x9b\x04\n\rEventTypeSpec\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x18\n\x10\x65vent_type_dcids\x18\x03 \x03(\t\x12\r\n\x05\x63olor\x18\x04 \x01(\t\x12<\n\x17\x64\x65\x66\x61ult_severity_filter\x18\x05 \x01(\x0b\x32\x1b.datacommons.SeverityFilter\x12[\n\x1aplace_type_severity_filter\x18\n \x03(\x0b\x32\x37.datacommons.EventTypeSpec.PlaceTypeSeverityFilterEntry\x12<\n\x0c\x64isplay_prop\x18\x06 \x03(\x0b\x32&.datacommons.EventTypeSpec.DisplayProp\x12\x15\n\rend_date_prop\x18\x07 \x03(\t\x12\x1d\n\x15polygon_geo_json_prop\x18\x08 \x01(\t\x12\x1a\n\x12path_geo_json_prop\x18\t \x01(\t\x1a[\n\x1cPlaceTypeSeverityFilterEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12*\n\x05value\x18\x02 \x01(\x0b\x32\x1b.datacommons.SeverityFilter:\x02\x38\x01\x1a?\n\x0b\x44isplayProp\x12\x0c\n\x04prop\x18\x01 \x01(\t\x12\x14\n\x0c\x64isplay_name\x18\x02 \x01(\t\x12\x0c\n\x04unit\x18\x03 \x01(\t\"\xe3\x03\n\x0cPageMetadata\x12\x10\n\x08topic_id\x18\x01 \x01(\t\x12\x12\n\ntopic_name\x18\x02 \x01(\t\x12\x12\n\nplace_dcid\x18\x03 \x03(\t\x12Q\n\x15\x63ontained_place_types\x18\x04 \x03(\x0b\x32\x32.datacommons.PageMetadata.ContainedPlaceTypesEntry\x12\x45\n\x0f\x65vent_type_spec\x18\x05 \x03(\x0b\x32,.datacommons.PageMetadata.EventTypeSpecEntry\x12\x39\n\x0bplace_group\x18\x06 \x03(\x0b\x32$.datacommons.PageMetadata.PlaceGroup\x1a:\n\x18\x43ontainedPlaceTypesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1aP\n\x12\x45ventTypeSpecEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12)\n\x05value\x18\x02 \x01(\x0b\x32\x1a.datacommons.EventTypeSpec:\x02\x38\x01\x1a\x36\n\nPlaceGroup\x12\x14\n\x0cparent_place\x18\x01 \x01(\t\x12\x12\n\nplace_type\x18\x02 \x01(\t\"\x9f\x01\n\x0bStatVarSpec\x12\x10\n\x08stat_var\x18\x01 \x01(\t\x12\r\n\x05\x64\x65nom\x18\x02 \x01(\t\x12\x0c\n\x04unit\x18\x03 \x01(\t\x12\x0f\n\x07scaling\x18\x04 \x01(\x01\x12\x0b\n\x03log\x18\x05 \x01(\x08\x12\x0c\n\x04name\x18\x06 \x01(\t\x12\x0c\n\x04\x64\x61te\x18\x07 \x01(\t\x12\x15\n\rno_per_capita\x18\x08 \x01(\x08\x12\x10\n\x08\x66\x61\x63\x65t_id\x18\t \x01(\t\"\xd0\x01\n\x0fRankingTileSpec\x12\x14\n\x0cshow_highest\x18\x01 \x01(\x08\x12\x13\n\x0bshow_lowest\x18\x02 \x01(\x08\x12\x16\n\x0e\x64iff_base_date\x18\x05 \x01(\t\x12\x15\n\rhighest_title\x18\x06 \x01(\t\x12\x14\n\x0clowest_title\x18\x07 \x01(\t\x12\x15\n\rranking_count\x18\n \x01(\x05\x12\x19\n\x11show_multi_column\x18\x0b \x01(\x08\x12\x1b\n\x13show_highest_lowest\x18\x0c \x01(\x08\"u\n\x18\x44isasterEventMapTileSpec\x12\x1c\n\x14point_event_type_key\x18\x01 \x03(\t\x12\x1e\n\x16polygon_event_type_key\x18\x02 \x03(\t\x12\x1b\n\x13path_event_type_key\x18\x03 \x03(\t\"9\n\x11HistogramTileSpec\x12\x16\n\x0e\x65vent_type_key\x18\x01 \x01(\t\x12\x0c\n\x04prop\x18\x02 \x01(\t\"\x9d\x01\n\x10TopEventTileSpec\x12\x16\n\x0e\x65vent_type_key\x18\x01 \x01(\t\x12\x14\n\x0c\x64isplay_prop\x18\x02 \x03(\t\x12\x17\n\x0fshow_start_date\x18\x03 \x01(\x08\x12\x15\n\rshow_end_date\x18\x04 \x01(\x08\x12\x14\n\x0creverse_sort\x18\x05 \x01(\x08\x12\x15\n\rranking_count\x18\x06 \x01(\x05\"\xbc\x01\n\x0fScatterTileSpec\x12\x1b\n\x13highlight_top_right\x18\x01 \x01(\x08\x12\x1a\n\x12highlight_top_left\x18\x02 \x01(\x08\x12\x1e\n\x16highlight_bottom_right\x18\x03 \x01(\x08\x12\x1d\n\x15highlight_bottom_left\x18\x04 \x01(\x08\x12\x19\n\x11show_place_labels\x18\x05 \x01(\x08\x12\x16\n\x0eshow_quadrants\x18\x06 \x01(\x08\"\xf0\x02\n\x0b\x42\x61rTileSpec\x12\x19\n\x11x_label_link_root\x18\x01 \x01(\t\x12\x12\n\nbar_height\x18\x02 \x01(\x01\x12\x0e\n\x06\x63olors\x18\x03 \x03(\t\x12\x12\n\nhorizontal\x18\x04 \x01(\x08\x12\x12\n\nmax_places\x18\x05 \x01(\x05\x12\x15\n\rmax_variables\x18\x06 \x01(\x05\x12/\n\x04sort\x18\x07 \x01(\x0e\x32!.datacommons.BarTileSpec.SortType\x12\x0f\n\x07stacked\x18\x08 \x01(\x08\x12\x14\n\x0cuse_lollipop\x18\t \x01(\x08\x12\x15\n\ry_axis_margin\x18\n \x01(\x01\"t\n\x08SortType\x12\x14\n\x10TYPE_UNSPECIFIED\x10\x00\x12\r\n\tASCENDING\x10\x01\x12\x0e\n\nDESCENDING\x10\x02\x12\x18\n\x14\x41SCENDING_POPULATION\x10\x03\x12\x19\n\x15\x44\x45SCENDING_POPULATION\x10\x04\"s\n\rGaugeTileSpec\x12/\n\x05range\x18\x01 \x01(\x0b\x32 .datacommons.GaugeTileSpec.Range\x12\x0e\n\x06\x63olors\x18\x02 \x03(\t\x1a!\n\x05Range\x12\x0b\n\x03min\x18\x01 \x01(\x01\x12\x0b\n\x03max\x18\x02 \x01(\x01\",\n\rDonutTileSpec\x12\x0e\n\x06\x63olors\x18\x01 \x03(\t\x12\x0b\n\x03pie\x18\x02 \x01(\x08\"\x1e\n\x0cLineTileSpec\x12\x0e\n\x06\x63olors\x18\x01 \x03(\t\"\x1d\n\x0bMapTileSpec\x12\x0e\n\x06\x63olors\x18\x02 \x03(\t\"\xf5\x07\n\x04Tile\x12\r\n\x05title\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12(\n\x04type\x18\x03 \x01(\x0e\x32\x1a.datacommons.Tile.TileType\x12\x14\n\x0cstat_var_key\x18\x04 \x03(\t\x12\x19\n\x11\x63omparison_places\x18\x07 \x03(\t\x12\x1b\n\x13place_dcid_override\x18\x0b \x01(\t\x12\x13\n\x0bhide_footer\x18\x11 \x01(\x08\x12\x39\n\x11ranking_tile_spec\x18\x05 \x01(\x0b\x32\x1c.datacommons.RankingTileSpecH\x00\x12M\n\x1c\x64isaster_event_map_tile_spec\x18\x06 \x01(\x0b\x32%.datacommons.DisasterEventMapTileSpecH\x00\x12<\n\x13top_event_tile_spec\x18\x08 \x01(\x0b\x32\x1d.datacommons.TopEventTileSpecH\x00\x12\x39\n\x11scatter_tile_spec\x18\t \x01(\x0b\x32\x1c.datacommons.ScatterTileSpecH\x00\x12=\n\x13histogram_tile_spec\x18\n \x01(\x0b\x32\x1e.datacommons.HistogramTileSpecH\x00\x12\x31\n\rbar_tile_spec\x18\x0c \x01(\x0b\x32\x18.datacommons.BarTileSpecH\x00\x12\x35\n\x0fgauge_tile_spec\x18\r \x01(\x0b\x32\x1a.datacommons.GaugeTileSpecH\x00\x12\x35\n\x0f\x64onut_tile_spec\x18\x0e \x01(\x0b\x32\x1a.datacommons.DonutTileSpecH\x00\x12\x33\n\x0eline_tile_spec\x18\x0f \x01(\x0b\x32\x19.datacommons.LineTileSpecH\x00\x12\x31\n\rmap_tile_spec\x18\x10 \x01(\x0b\x32\x18.datacommons.MapTileSpecH\x00\"\xde\x01\n\x08TileType\x12\r\n\tTYPE_NONE\x10\x00\x12\x08\n\x04LINE\x10\x01\x12\x07\n\x03\x42\x41R\x10\x02\x12\x07\n\x03MAP\x10\x03\x12\x0b\n\x07SCATTER\x10\x04\x12\r\n\tBIVARIATE\x10\x05\x12\x0b\n\x07RANKING\x10\x06\x12\r\n\tHIGHLIGHT\x10\x07\x12\x0f\n\x0b\x44\x45SCRIPTION\x10\x08\x12\t\n\x05GAUGE\x10\r\x12\t\n\x05\x44ONUT\x10\x0e\x12\r\n\tHISTOGRAM\x10\n\x12\x12\n\x0ePLACE_OVERVIEW\x10\x0b\x12\r\n\tTOP_EVENT\x10\x0c\x12\x16\n\x12\x44ISASTER_EVENT_MAP\x10\tB\x10\n\x0etile_type_spec\"\xcf\x01\n\x11\x44isasterBlockSpec\x12>\n\ndate_range\x18\x01 \x01(\x0e\x32(.datacommons.DisasterBlockSpec.DateRangeH\x00\x12\x0e\n\x04\x64\x61te\x18\x02 \x01(\tH\x00\"Z\n\tDateRange\x12\r\n\tTYPE_NONE\x10\x00\x12\x0f\n\x0bTHIRTY_DAYS\x10\x01\x12\x0e\n\nSIX_MONTHS\x10\x02\x12\x0c\n\x08ONE_YEAR\x10\x03\x12\x0f\n\x0bTHREE_YEARS\x10\x04\x42\x0e\n\x0c\x64\x65\x66\x61ult_date\"\xec\x02\n\x05\x42lock\x12\r\n\x05title\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x10\n\x08\x66ootnote\x18\x05 \x01(\t\x12*\n\x07\x63olumns\x18\x03 \x03(\x0b\x32\x19.datacommons.Block.Column\x12*\n\x04type\x18\x04 \x01(\x0e\x32\x1c.datacommons.Block.BlockType\x12\r\n\x05\x64\x65nom\x18\x06 \x01(\t\x12\x18\n\x10start_with_denom\x18\x07 \x01(\x08\x12=\n\x13\x64isaster_block_spec\x18\x08 \x01(\x0b\x32\x1e.datacommons.DisasterBlockSpecH\x00\x1a*\n\x06\x43olumn\x12 \n\x05tiles\x18\x01 \x03(\x0b\x32\x11.datacommons.Tile\".\n\tBlockType\x12\r\n\tTYPE_NONE\x10\x00\x12\x12\n\x0e\x44ISASTER_EVENT\x10\x01\x42\x11\n\x0f\x62lock_type_spec\"\xed\x01\n\x08\x43\x61tegory\x12\r\n\x05title\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12=\n\rstat_var_spec\x18\x04 \x03(\x0b\x32&.datacommons.Category.StatVarSpecEntry\x12\"\n\x06\x62locks\x18\x03 \x03(\x0b\x32\x12.datacommons.Block\x12\x0c\n\x04\x64\x63id\x18\x05 \x01(\t\x1aL\n\x10StatVarSpecEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\'\n\x05value\x18\x02 \x01(\x0b\x32\x18.datacommons.StatVarSpec:\x02\x38\x01\"q\n\x11SubjectPageConfig\x12+\n\x08metadata\x18\x01 \x01(\x0b\x32\x19.datacommons.PageMetadata\x12)\n\ncategories\x18\x02 \x03(\x0b\x32\x15.datacommons.CategoryJ\x04\x08\x03\x10\x04\x62\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'subject_page_pb2', globals()) @@ -71,23 +71,23 @@ _MAPTILESPEC._serialized_start=2641 _MAPTILESPEC._serialized_end=2670 _TILE._serialized_start=2673 - _TILE._serialized_end=3665 - _TILE_TILETYPE._serialized_start=3425 - _TILE_TILETYPE._serialized_end=3647 - _DISASTERBLOCKSPEC._serialized_start=3668 - _DISASTERBLOCKSPEC._serialized_end=3875 - _DISASTERBLOCKSPEC_DATERANGE._serialized_start=3769 - _DISASTERBLOCKSPEC_DATERANGE._serialized_end=3859 - _BLOCK._serialized_start=3878 - _BLOCK._serialized_end=4242 - _BLOCK_COLUMN._serialized_start=4133 - _BLOCK_COLUMN._serialized_end=4175 - _BLOCK_BLOCKTYPE._serialized_start=4177 - _BLOCK_BLOCKTYPE._serialized_end=4223 - _CATEGORY._serialized_start=4245 - _CATEGORY._serialized_end=4482 - _CATEGORY_STATVARSPECENTRY._serialized_start=4406 - _CATEGORY_STATVARSPECENTRY._serialized_end=4482 - _SUBJECTPAGECONFIG._serialized_start=4484 - _SUBJECTPAGECONFIG._serialized_end=4597 + _TILE._serialized_end=3686 + _TILE_TILETYPE._serialized_start=3446 + _TILE_TILETYPE._serialized_end=3668 + _DISASTERBLOCKSPEC._serialized_start=3689 + _DISASTERBLOCKSPEC._serialized_end=3896 + _DISASTERBLOCKSPEC_DATERANGE._serialized_start=3790 + _DISASTERBLOCKSPEC_DATERANGE._serialized_end=3880 + _BLOCK._serialized_start=3899 + _BLOCK._serialized_end=4263 + _BLOCK_COLUMN._serialized_start=4154 + _BLOCK_COLUMN._serialized_end=4196 + _BLOCK_BLOCKTYPE._serialized_start=4198 + _BLOCK_BLOCKTYPE._serialized_end=4244 + _CATEGORY._serialized_start=4266 + _CATEGORY._serialized_end=4503 + _CATEGORY_STATVARSPECENTRY._serialized_start=4427 + _CATEGORY_STATVARSPECENTRY._serialized_end=4503 + _SUBJECTPAGECONFIG._serialized_start=4505 + _SUBJECTPAGECONFIG._serialized_end=4618 # @@protoc_insertion_point(module_scope) diff --git a/server/integration_tests/test_data/detection_api_statvars/correlatewithgdpofcalifornia/chart_config.json b/server/integration_tests/test_data/detection_api_statvars/correlatewithgdpofcalifornia/chart_config.json index f2c7f063cd..29d466d901 100644 --- a/server/integration_tests/test_data/detection_api_statvars/correlatewithgdpofcalifornia/chart_config.json +++ b/server/integration_tests/test_data/detection_api_statvars/correlatewithgdpofcalifornia/chart_config.json @@ -39,8 +39,8 @@ "USStateQuarterlyIndustryGDP_NAICS_44_45", "dc/topic/Economy", "dc/topic/EconomicEquity", - "dc/topic/GDP", "Amount_EconomicActivity_GrossDomesticProduction_RealValue", + "dc/topic/GDP", "Amount_EconomicActivity_GrossDomesticProduction_Nominal_AsAFractionOf_Count_Person" ] } \ No newline at end of file diff --git a/server/integration_tests/test_data/translate_hindi/query_1/chart_config.json b/server/integration_tests/test_data/translate_hindi/query_1/chart_config.json index 7368fa008f..a34f5a1266 100644 --- a/server/integration_tests/test_data/translate_hindi/query_1/chart_config.json +++ b/server/integration_tests/test_data/translate_hindi/query_1/chart_config.json @@ -3,6 +3,230 @@ "categories": [ { "blocks": [ + { + "columns": [ + { + "tiles": [ + { + "rankingTileSpec": { + "rankingCount": 10, + "showHighest": true + }, + "statVarKey": [ + "Count_CriminalActivities_Robbery" + ], + "title": "Robberies in Cities of Santa Clara County (${date})", + "type": "RANKING" + } + ] + }, + { + "tiles": [ + { + "statVarKey": [ + "Count_CriminalActivities_Robbery" + ], + "title": "Robberies in Cities of Santa Clara County (${date})", + "type": "MAP" + } + ] + } + ], + "denom": "Count_Person", + "title": "Robberies in Cities of Santa Clara County" + }, + { + "columns": [ + { + "tiles": [ + { + "rankingTileSpec": { + "rankingCount": 10, + "showHighest": true + }, + "statVarKey": [ + "Count_CriminalActivities_PropertyCrime" + ], + "title": "Property Related Crimes in Cities of Santa Clara County (${date})", + "type": "RANKING" + } + ] + }, + { + "tiles": [ + { + "statVarKey": [ + "Count_CriminalActivities_PropertyCrime" + ], + "title": "Property Related Crimes in Cities of Santa Clara County (${date})", + "type": "MAP" + } + ] + } + ], + "denom": "Count_Person", + "title": "Property Related Crimes in Cities of Santa Clara County" + }, + { + "columns": [ + { + "tiles": [ + { + "rankingTileSpec": { + "rankingCount": 10, + "showHighest": true + }, + "statVarKey": [ + "Count_CriminalActivities_LarcenyTheft" + ], + "title": "Larceny Theft Cases in Cities of Santa Clara County (${date})", + "type": "RANKING" + } + ] + }, + { + "tiles": [ + { + "statVarKey": [ + "Count_CriminalActivities_LarcenyTheft" + ], + "title": "Larceny Theft Cases in Cities of Santa Clara County (${date})", + "type": "MAP" + } + ] + } + ], + "denom": "Count_Person", + "title": "Larceny Theft Cases in Cities of Santa Clara County" + }, + { + "columns": [ + { + "tiles": [ + { + "rankingTileSpec": { + "rankingCount": 10, + "showHighest": true + }, + "statVarKey": [ + "Count_CriminalActivities_MotorVehicleTheft" + ], + "title": "Motor Vehicle Thefts in Cities of Santa Clara County (${date})", + "type": "RANKING" + } + ] + }, + { + "tiles": [ + { + "statVarKey": [ + "Count_CriminalActivities_MotorVehicleTheft" + ], + "title": "Motor Vehicle Thefts in Cities of Santa Clara County (${date})", + "type": "MAP" + } + ] + } + ], + "denom": "Count_Person", + "title": "Motor Vehicle Thefts in Cities of Santa Clara County" + }, + { + "columns": [ + { + "tiles": [ + { + "rankingTileSpec": { + "rankingCount": 10, + "showHighest": true + }, + "statVarKey": [ + "Count_CriminalActivities_CombinedCrime" + ], + "title": "Criminal Activities in Cities of Santa Clara County (${date})", + "type": "RANKING" + } + ] + }, + { + "tiles": [ + { + "statVarKey": [ + "Count_CriminalActivities_CombinedCrime" + ], + "title": "Criminal Activities in Cities of Santa Clara County (${date})", + "type": "MAP" + } + ] + } + ], + "denom": "Count_Person", + "title": "Criminal Activities in Cities of Santa Clara County" + }, + { + "columns": [ + { + "tiles": [ + { + "rankingTileSpec": { + "rankingCount": 10, + "showHighest": true + }, + "statVarKey": [ + "Count_CriminalActivities_AggravatedAssault" + ], + "title": "Aggravated Assault Cases in Cities of Santa Clara County (${date})", + "type": "RANKING" + } + ] + }, + { + "tiles": [ + { + "statVarKey": [ + "Count_CriminalActivities_AggravatedAssault" + ], + "title": "Aggravated Assault Cases in Cities of Santa Clara County (${date})", + "type": "MAP" + } + ] + } + ], + "denom": "Count_Person", + "title": "Aggravated Assault Cases in Cities of Santa Clara County" + }, + { + "columns": [ + { + "tiles": [ + { + "rankingTileSpec": { + "rankingCount": 10, + "showHighest": true + }, + "statVarKey": [ + "Count_CriminalActivities_Arson" + ], + "title": "Arson Crimes in Cities of Santa Clara County (${date})", + "type": "RANKING" + } + ] + }, + { + "tiles": [ + { + "statVarKey": [ + "Count_CriminalActivities_Arson" + ], + "title": "Arson Crimes in Cities of Santa Clara County (${date})", + "type": "MAP" + } + ] + } + ], + "denom": "Count_Person", + "title": "Arson Crimes in Cities of Santa Clara County" + }, { "columns": [ { @@ -34,12 +258,112 @@ ], "denom": "Count_Person", "title": "Burglaries in Cities of Santa Clara County" + }, + { + "columns": [ + { + "tiles": [ + { + "rankingTileSpec": { + "rankingCount": 10, + "showHighest": true + }, + "statVarKey": [ + "Count_CriminalActivities_ForcibleRape" + ], + "title": "Forcible Rapes in Cities of Santa Clara County (${date})", + "type": "RANKING" + } + ] + }, + { + "tiles": [ + { + "statVarKey": [ + "Count_CriminalActivities_ForcibleRape" + ], + "title": "Forcible Rapes in Cities of Santa Clara County (${date})", + "type": "MAP" + } + ] + } + ], + "denom": "Count_Person", + "title": "Forcible Rapes in Cities of Santa Clara County" + }, + { + "columns": [ + { + "tiles": [ + { + "rankingTileSpec": { + "rankingCount": 10, + "showHighest": true + }, + "statVarKey": [ + "Count_CriminalActivities_ViolentCrime" + ], + "title": "Violent Crimes in Cities of Santa Clara County (${date})", + "type": "RANKING" + } + ] + }, + { + "tiles": [ + { + "statVarKey": [ + "Count_CriminalActivities_ViolentCrime" + ], + "title": "Violent Crimes in Cities of Santa Clara County (${date})", + "type": "MAP" + } + ] + } + ], + "denom": "Count_Person", + "title": "Violent Crimes in Cities of Santa Clara County" } ], "statVarSpec": { + "Count_CriminalActivities_AggravatedAssault": { + "name": "Aggravated Assault Cases", + "statVar": "Count_CriminalActivities_AggravatedAssault" + }, + "Count_CriminalActivities_Arson": { + "name": "Arson Crimes", + "statVar": "Count_CriminalActivities_Arson" + }, "Count_CriminalActivities_Burglary": { "name": "Burglaries", "statVar": "Count_CriminalActivities_Burglary" + }, + "Count_CriminalActivities_CombinedCrime": { + "name": "Criminal Activities", + "statVar": "Count_CriminalActivities_CombinedCrime" + }, + "Count_CriminalActivities_ForcibleRape": { + "name": "Forcible Rapes", + "statVar": "Count_CriminalActivities_ForcibleRape" + }, + "Count_CriminalActivities_LarcenyTheft": { + "name": "Larceny Theft Cases", + "statVar": "Count_CriminalActivities_LarcenyTheft" + }, + "Count_CriminalActivities_MotorVehicleTheft": { + "name": "Motor Vehicle Thefts", + "statVar": "Count_CriminalActivities_MotorVehicleTheft" + }, + "Count_CriminalActivities_PropertyCrime": { + "name": "Property Related Crimes", + "statVar": "Count_CriminalActivities_PropertyCrime" + }, + "Count_CriminalActivities_Robbery": { + "name": "Robberies", + "statVar": "Count_CriminalActivities_Robbery" + }, + "Count_CriminalActivities_ViolentCrime": { + "name": "Violent Crimes", + "statVar": "Count_CriminalActivities_ViolentCrime" } } } diff --git a/server/routes/import_wizard/html.py b/server/routes/import_wizard/html.py index 741f1b8aa1..08bb955227 100644 --- a/server/routes/import_wizard/html.py +++ b/server/routes/import_wizard/html.py @@ -14,10 +14,12 @@ """Import Wizard routes""" import os +import subprocess -import flask from flask import Blueprint +from flask import jsonify from flask import render_template +from flask import request bp = Blueprint('import_wizard', __name__, url_prefix='/import') @@ -30,3 +32,60 @@ def main(): @bp.route('/') def main_new(): return render_template('/import_wizard2.html') + + +# This is only used in CSV + SQL backend, converting raw CSV data into resolved +# CSV. +@bp.route('/simple/convert', methods=['POST']) +def convert(): + raw_data_path = request.json.get("rawDataPath") or os.environ.get( + "RAW_DATA_PATH") + sql_data_path = request.json.get("sqlDataPath") or os.environ.get( + "SQL_DATA_PATH") + command1 = [ + "python", + "import/simple/stats/main.py", + "--input_path", + f"{raw_data_path}/input.csv", # TODO: use gcs and local folder path. + "--output_dir", + f"{sql_data_path}", + ] + # TODO: save debug information in another folder + command2 = ["rm", f"{sql_data_path}/debug_resolve.csv"] + output = [] + for command in [command1, command2]: + try: + result = subprocess.run(command, capture_output=True, text=True) + if result.returncode == 0: + output.append({"status": "success", "output": result.stdout.strip()}) + else: + return jsonify({ + "status": "failure", + "error": result.stderr.strip() + }), 500 + except Exception as e: + return jsonify({"status": "error", "message": str(e)}), 500 + return jsonify(output), 200 + + +# This is a proxy to trigger Mixer import for csv + SQL backend. +# This can only be used for bundled mixer and website in the same docker image. +@bp.route('/simple/load2sql', methods=['POST']) +def load2sql(): + command = [ + "curl", + "-X", + "POST", + "localhost:8081/import", + ] + try: + result = subprocess.run(command, capture_output=True, text=True) + if result.returncode == 0: + return jsonify({ + "status": "success", + "output": result.stdout.strip() + }), 200 + else: + return jsonify({"status": "failure", "error": result.stderr.strip()}), 500 + except Exception as e: + return jsonify({"status": "error", "message": str(e)}), 500 \ No newline at end of file