remove per capita stop words (#4456)

same change as #4415 but rebased off a clean master svindex diff: https://storage.mtls.cloud.google.com/datcom-embedding-diffs/chejennifer_base_uae_mem_2024_07_09_21_55_14.html base does not remove per capita stop words test removes per capita stop words
datacommonsorg · Jul 12, 2024 · 95e7096 · 95e7096
1 parent 10a081d
commit 95e7096
Show file tree

Hide file tree

Showing 22 changed files with 124 additions and 134 deletions.
diff --git a/deploy/nl/catalog.yaml b/deploy/nl/catalog.yaml
@@ -90,7 +90,7 @@ indexes:
   base_uae_mem:
     store_type: MEMORY
     source_path: ../../tools/nl/embeddings/input/base
-    embeddings_path: gs://datcom-nl-models/base_uae_mem_2024_07_11_08_35_57/embeddings.csv
+    embeddings_path: gs://datcom-nl-models/base_uae_mem_2024_07_12_09_03_25/embeddings.csv
     model: uae-large-v1-model
     healthcheck_query: "Life expectancy"
   base_mistral_mem:

diff --git a/run_test.sh b/run_test.sh
@@ -64,7 +64,7 @@ function run_lint_fix {
     pip3 install isort -q
   fi
   yapf -r -i -p --style='{based_on_style: google, indent_width: 2}' server/ nl_server/ shared/ tools/ -e=*pb2.py -e=**/.env/**
-  isort server/ nl_server/ shared/ tools/  --skip-glob *pb2.py  --skip-glob **/.env/** --profile google
+  isort server/ nl_server/ shared/ tools/  --skip-glob=*pb2.py  --skip-glob=**/.env/** --profile=google
   deactivate
 }
 

diff --git a/server/integration_tests/test_data/demo_fallback/query_2/chart_config.json b/server/integration_tests/test_data/demo_fallback/query_2/chart_config.json
@@ -78,16 +78,16 @@
                       "geoId/12"
                     ],
                     "statVarKey": [
-                      "Count_CriminalIncidents_IsHateCrime_multiple_place_bar_block"
+                      "Count_CriminalActivities_MurderAndNonNegligentManslaughter_multiple_place_bar_block"
                     ],
-                    "title": "Hate Crime Incidents (${date})",
+                    "title": "Murder and Non Negligent Manslaughter Cases (${date})",
                     "type": "BAR"
                   }
                 ]
               }
             ],
             "denom": "Count_Person",
-            "title": "Hate Crime Incidents"
+            "title": "Murder and Non Negligent Manslaughter Cases"
           },
           {
             "columns": [
@@ -104,17 +104,16 @@
                       "geoId/12"
                     ],
                     "statVarKey": [
-                      "Count_CriminalIncidents_BiasMotivationDisabilityStatus_AggravatedAssault_IsHateCrime_multiple_place_bar_block",
-                      "Count_CriminalIncidents_BiasMotivationDisabilityStatus_Robbery_IsHateCrime_multiple_place_bar_block"
+                      "Count_CriminalIncidents_IsHateCrime_multiple_place_bar_block"
                     ],
-                    "title": "Hate Crime Cases by Type (${date})",
+                    "title": "Hate Crime Incidents (${date})",
                     "type": "BAR"
                   }
                 ]
               }
             ],
             "denom": "Count_Person",
-            "title": "Hate Crime Cases by Type"
+            "title": "Hate Crime Incidents"
           },
           {
             "columns": [
@@ -131,16 +130,17 @@
                       "geoId/12"
                     ],
                     "statVarKey": [
-                      "Count_CriminalActivities_MurderAndNonNegligentManslaughter_multiple_place_bar_block"
+                      "Count_CriminalIncidents_BiasMotivationDisabilityStatus_AggravatedAssault_IsHateCrime_multiple_place_bar_block",
+                      "Count_CriminalIncidents_BiasMotivationDisabilityStatus_Robbery_IsHateCrime_multiple_place_bar_block"
                     ],
-                    "title": "Murder and Non Negligent Manslaughter Cases (${date})",
+                    "title": "Hate Crime Cases by Type (${date})",
                     "type": "BAR"
                   }
                 ]
               }
             ],
             "denom": "Count_Person",
-            "title": "Murder and Non Negligent Manslaughter Cases"
+            "title": "Hate Crime Cases by Type"
           },
           {
             "columns": [

diff --git a/server/integration_tests/test_data/demo_fallback/query_3/chart_config.json b/server/integration_tests/test_data/demo_fallback/query_3/chart_config.json
@@ -71,76 +71,76 @@
                 "tiles": [
                   {
                     "statVarKey": [
-                      "Count_CriminalIncidents_IsHateCrime"
+                      "Count_CriminalActivities_MurderAndNonNegligentManslaughter"
                     ],
-                    "title": "Hate Crime Incidents in California",
+                    "title": "Murder and Non Negligent Manslaughter Cases in California",
                     "type": "LINE"
                   }
                 ]
               },
               {
                 "tiles": [
                   {
-                    "description": "Hate Crime Incidents in California",
+                    "description": "Murder and Non Negligent Manslaughter Cases in California",
                     "statVarKey": [
-                      "Count_CriminalIncidents_IsHateCrime"
+                      "Count_CriminalActivities_MurderAndNonNegligentManslaughter"
                     ],
-                    "title": "Hate Crime Incidents in California",
+                    "title": "Murder and Non Negligent Manslaughter Cases in California",
                     "type": "HIGHLIGHT"
                   }
                 ]
               }
             ],
             "denom": "Count_Person",
-            "title": "Hate Crime Incidents"
+            "title": "Murder and Non Negligent Manslaughter Cases"
           },
           {
             "columns": [
               {
                 "tiles": [
                   {
                     "statVarKey": [
-                      "Count_CriminalIncidents_BiasMotivationDisabilityStatus_AggravatedAssault_IsHateCrime",
-                      "Count_CriminalIncidents_BiasMotivationDisabilityStatus_Intimidation_IsHateCrime",
-                      "Count_CriminalIncidents_BiasMotivationDisabilityStatus_Robbery_IsHateCrime"
+                      "Count_CriminalIncidents_IsHateCrime"
                     ],
-                    "title": "Hate Crime Cases by Type in California",
+                    "title": "Hate Crime Incidents in California",
                     "type": "LINE"
                   }
                 ]
-              }
-            ],
-            "denom": "Count_Person",
-            "title": "Hate Crime Cases by Type"
-          },
-          {
-            "columns": [
+              },
               {
                 "tiles": [
                   {
+                    "description": "Hate Crime Incidents in California",
                     "statVarKey": [
-                      "Count_CriminalActivities_MurderAndNonNegligentManslaughter"
+                      "Count_CriminalIncidents_IsHateCrime"
                     ],
-                    "title": "Murder and Non Negligent Manslaughter Cases in California",
-                    "type": "LINE"
+                    "title": "Hate Crime Incidents in California",
+                    "type": "HIGHLIGHT"
                   }
                 ]
-              },
+              }
+            ],
+            "denom": "Count_Person",
+            "title": "Hate Crime Incidents"
+          },
+          {
+            "columns": [
               {
                 "tiles": [
                   {
-                    "description": "Murder and Non Negligent Manslaughter Cases in California",
                     "statVarKey": [
-                      "Count_CriminalActivities_MurderAndNonNegligentManslaughter"
+                      "Count_CriminalIncidents_BiasMotivationDisabilityStatus_AggravatedAssault_IsHateCrime",
+                      "Count_CriminalIncidents_BiasMotivationDisabilityStatus_Intimidation_IsHateCrime",
+                      "Count_CriminalIncidents_BiasMotivationDisabilityStatus_Robbery_IsHateCrime"
                     ],
-                    "title": "Murder and Non Negligent Manslaughter Cases in California",
-                    "type": "HIGHLIGHT"
+                    "title": "Hate Crime Cases by Type in California",
+                    "type": "LINE"
                   }
                 ]
               }
             ],
             "denom": "Count_Person",
-            "title": "Murder and Non Negligent Manslaughter Cases"
+            "title": "Hate Crime Cases by Type"
           },
           {
             "columns": [

diff --git a/...est_data/detection_api_multivar/comparemalepopulationwithfemalepopulation/debug_info.json b/...est_data/detection_api_multivar/comparemalepopulationwithfemalepopulation/debug_info.json
@@ -54,7 +54,7 @@
           "Parts": [
             {
               "CosineScore": [
-                0.9257857799530029
+                0.9257858991622925
               ],
               "QueryPart": "male population",
               "SV": [
@@ -63,8 +63,8 @@
             },
             {
               "CosineScore": [
-                0.9296980500221252,
-                0.8847433924674988
+                0.9296978712081909,
+                0.884743332862854
               ],
               "QueryPart": "female population",
               "SV": [
@@ -80,7 +80,7 @@
           "Parts": [
             {
               "CosineScore": [
-                0.9117770791053772
+                0.911777138710022
               ],
               "QueryPart": "male population female",
               "SV": [
@@ -89,8 +89,8 @@
             },
             {
               "CosineScore": [
-                0.8982378840446472,
-                0.8723467588424683
+                0.8982377648353577,
+                0.8723466396331787
               ],
               "QueryPart": "population",
               "SV": [
@@ -106,9 +106,9 @@
           "Parts": [
             {
               "CosineScore": [
-                0.8034241199493408,
-                0.8017654418945312,
-                0.775155782699585,
+                0.8034241795539856,
+                0.8017653822898865,
+                0.7751558423042297,
                 0.7698014974594116,
                 0.7586972713470459
               ],

diff --git a/...ar/howarefactorslikeobesity,bloodpressureandasthmaimpactedbyclimatechange/debug_info.json b/...ar/howarefactorslikeobesity,bloodpressureandasthmaimpactedbyclimatechange/debug_info.json
@@ -113,10 +113,10 @@
           "Parts": [
             {
               "CosineScore": [
-                0.7768431901931763,
-                0.7495046257972717,
-                0.7342654466629028,
-                0.7325114011764526
+                0.7768430709838867,
+                0.749504566192627,
+                0.7342653274536133,
+                0.7325113415718079
               ],
               "QueryPart": "factors like obesity blood pressure asthma",
               "SV": [
@@ -128,7 +128,7 @@
             },
             {
               "CosineScore": [
-                0.9056920409202576
+                0.9056921005249023
               ],
               "QueryPart": "impacted climate change",
               "SV": [

diff --git a/server/integration_tests/test_data/detection_translate_chinese/chart_config.json b/server/integration_tests/test_data/detection_translate_chinese/chart_config.json
@@ -11,9 +11,6 @@
       "contained_in_place_type": "City",
       "had_default_type": false,
       "type": 4
-    },
-    {
-      "type": 14
     }
   ],
   "client": "test_detect",

diff --git a/...t_data/e2e_edge_cases2/povertyvs.unemploymentrateindistrictsoftamilnadu/chart_config.json b/...t_data/e2e_edge_cases2/povertyvs.unemploymentrateindistrictsoftamilnadu/chart_config.json
@@ -31,7 +31,6 @@
               }
             ],
             "denom": "Count_Person",
-            "startWithDenom": true,
             "title": "Population Below Poverty Line"
           },
           {

diff --git a/server/integration_tests/test_data/e2e_high_sv_threshold/chart_config.json b/server/integration_tests/test_data/e2e_high_sv_threshold/chart_config.json
@@ -62,7 +62,6 @@
               }
             ],
             "denom": "Count_Person",
-            "startWithDenom": true,
             "title": "Count of Mortality Event: 1 Years or Less in Counties of Massachusetts"
           },
           {
@@ -83,7 +82,6 @@
               }
             ],
             "denom": "Count_Person",
-            "startWithDenom": true,
             "title": "Causes of Infant Mortality"
           },
           {
@@ -116,7 +114,6 @@
               }
             ],
             "denom": "Count_Person",
-            "startWithDenom": true,
             "title": "Mortality Events (1 Years or Less): Symptoms, Signs and Abnormal Clinical and Laboratory Findings, Not Elsewhere Classified) in Counties of Massachusetts"
           },
           {
@@ -149,7 +146,6 @@
               }
             ],
             "denom": "Count_Person",
-            "startWithDenom": true,
             "title": "Mortality Events (1 Years or Less): Certain Conditions Originating in the Perinatal Period) in Counties of Massachusetts"
           },
           {

diff --git a/...sts/test_data/e2e_india_demo/howdoesliteracyratecomparetopovertyinindia/chart_config.json b/...sts/test_data/e2e_india_demo/howdoesliteracyratecomparetopovertyinindia/chart_config.json
@@ -23,7 +23,6 @@
               }
             ],
             "denom": "Count_Person",
-            "startWithDenom": true,
             "title": "Population: Literate vs. Population Below poverty line"
           },
           {
@@ -56,7 +55,6 @@
               }
             ],
             "denom": "Count_Person",
-            "startWithDenom": true,
             "title": "Population: Literate in Administrative Area 1 Places of India"
           },
           {
@@ -89,7 +87,6 @@
               }
             ],
             "denom": "Count_Person",
-            "startWithDenom": true,
             "title": "Population Below Poverty Line in Administrative Area 1 Places of India"
           },
           {
@@ -111,7 +108,6 @@
               }
             ],
             "denom": "Count_Person",
-            "startWithDenom": true,
             "title": "Population: Illiterate vs. Population Below poverty line (Per Capita)"
           },
           {
@@ -195,7 +191,6 @@
               }
             ],
             "denom": "Count_Person",
-            "startWithDenom": true,
             "title": "Population: Literate vs. Population Below poverty line (Per Capita)"
           },
           {
@@ -217,7 +212,6 @@
               }
             ],
             "denom": "Count_Person",
-            "startWithDenom": true,
             "title": "Population: Illiterate vs. Population Below poverty line"
           }
         ],
-Original file line number
+Diff line change
@@ Expand Up / @@ -31,7 +31,6 @@ @@
                   }
                 ],
                 "denom": "Count_Person",
-                "startWithDenom": true,
                 "title": "Population Below Poverty Line"
               },
               {
@@ Expand Down @@