Bundle import tool into RSI docker image; Add website endpoint for RS…

…I data load (#3667)

.gitignore

@@ -66,6 +66,7 @@ experimental/sdg-static/datacommons/nl_interface.min.css
 
 # Custom DC data
 data/
+dc-data/
 
 # Topic cache
 gen_ordered_list_for_topics.mcf

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "mixer"]
 	path = mixer
-	url = https://github.com/datacommonsorg/mixer.git
+	url = https://github.com/datacommonsorg/mixer.git
+[submodule "import"]
+	path = import
+	url = https://github.com/datacommonsorg/import.git

build/python_node_go_protoc_envoy/Dockerfile

@@ -16,7 +16,7 @@
 FROM python:3.11.4-slim as base
 
 RUN apt update
-RUN apt install -y curl zip wget
+RUN apt install -y curl zip wget gcc libc6-dev
 
 # Install Node
 ENV NODE_VERSION=18.17.1

build/python_node_go_protoc_envoy/cloudbuild.yaml

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 substitutions:
-  _VERS: "2023-08-19"
+  _VERS: "2023-10-05"
 
 steps:
   - name: "gcr.io/cloud-builders/docker"

build/web_compose/Dockerfile

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 
-FROM gcr.io/datcom-ci/python-node-go-protoc-envoy:2023-08-19
+FROM gcr.io/datcom-ci/python-node-go-protoc-envoy:2023-10-05
 
 ARG ENV
 ENV ENV=${ENV}
@@ -48,6 +48,7 @@ RUN GRPC_HEALTH_PROBE_VERSION=v0.4.7 && \
 # Build binary
 COPY mixer/cmd/ cmd
 COPY mixer/esp/ esp
+ENV CGO_ENABLED=1
 RUN go build -o /go/bin/mixer cmd/main.go
 
 # Flask
@@ -57,6 +58,8 @@ RUN pip install --upgrade pip
 RUN pip install --upgrade setuptools
 # --no-cache-dir removes ~/.cache files, which can be a few GB.
 RUN pip3 install --no-cache-dir -r /workspace/server/requirements.txt
+# Used by Import Tool
+RUN pip install pandas
 
 # NPM
 WORKDIR /workspace/static
@@ -72,6 +75,10 @@ RUN npm run-script build
 COPY server/. /workspace/server/
 COPY shared/. /workspace/shared/
 
+# Import Tool
+COPY import/. /workspace/import/
+
+# Env
 ENV USE_SQLITE=false
 ENV USE_CLOUDSQL=false
 ENV REMOTE_MIXER_DOMAIN=https://api.datacommons.org

docs/custom_dc.md

@@ -82,22 +82,21 @@ docker run -it --pull=always \
 -e USE_SQLITE=true \
 -e SQL_DATA_PATH=/sqlite \
 -p 8080:8080 \
--p 8081:8081 \
 -v $HOME/dc-data:/sqlite \
 gcr.io/datcom-ci/datacommons-website-compose:latest
 ```
 
+Run `curl -X POST localhost:8080/import/simple/load2sql` to load the data into
+database. Whenever there is update on the data, re-run this commands (no need to
+restart the container).
+
 If you have your own UI updates and build the docker image locally, replace the
 docker image with your locally one in the command.
 
 Now you can access a custom Data Commons site via
 [localhost](http://localhost:8080). For example, the data from the sample data
 can be viewed in [Timeline Chart](http://localhost:8080/tools/timeline#place=geoId%2F06&statsVar=stat_var_1).
 
-If the CSV files are updated, just run `curl -X POST localhost:8081/import`.
-This re-imports the CSV files into the local instance without the need to
-restart the container.
-
 ## Run in Cloud
 
 Custom Data Commons can be ran on the cloud as a production service. The SQLite
@@ -148,7 +147,6 @@ docker run -it \
 -e GOOGLE_APPLICATION_CREDENTIALS=/gcp/creds.json \
 -v $HOME/.config/gcloud/application_default_credentials.json:/gcp/creds.json:ro \
 -p 8080:8080 \
--p 8081:8081 \
 gcr.io/datcom-ci/datacommons-website-compose:latest
 ```
 

server/integration_tests/test_data/detection_api_statvars/correlatewithgdpofcalifornia/chart_config.json

@@ -39,8 +39,8 @@
     "USStateQuarterlyIndustryGDP_NAICS_44_45",
     "dc/topic/Economy",
     "dc/topic/EconomicEquity",
-    "dc/topic/GDP",
     "Amount_EconomicActivity_GrossDomesticProduction_RealValue",
+    "dc/topic/GDP",
     "Amount_EconomicActivity_GrossDomesticProduction_Nominal_AsAFractionOf_Count_Person"
   ]
 }