Dev sync ago

sample_config.py

@@ -4,19 +4,10 @@
 LOG_LEVEL               = 'INFO'
 LOG_PATH                = './logs/sync.log'
 
-# Secrets for sync-ago.py
-MAPS_PASSWORD = 'password'
-SALESFORCE_AGO_ITEMID = '1234abcdf'
-DATABRIDGE_SDE_PASSWORD = 'password'
-DATABRIDGE_HOST = 'database.host'
-DATABRIDGE_DB = 'database'
-
-THREEONEONE_PASSWORD = 'password'
 
 # These describe the destination (enterprise) dataset.
 # These describe the destination (enterprise) dataset.
-DEST_PROD_DSN = ''
-DEST_TEST_DSN = ''
+
 DEST_DB_ACCOUNT         = ''
 DEST_TABLE              = 'SALESFORCE_CASES'
 DEST_UPDATED_FIELD      = 'updated_datetime'
@@ -61,10 +52,7 @@
 # Microsoft Teams Web Connector URL
 MSTEAMS_CONNECTOR             = 'https://phila.webhook.office.com/webhookb2/763c9a83-0f38-4eb2-abfc-e0f2f41b6fbb@2046864f-68ea-497d-af34-a6629a6cd700/IncomingWebhook/434dfb8c116d472f8f224cfae367cdc1/2f82d684-85a4-4131-95a3-3342e012faeb'
 
-# These are for querying Salesforce.
-SF_USER             = ''
-SF_PASSWORD         = ''
-SF_TOKEN            = ''
+
 # Most of the filtering for the public view we do in the database, but the
 # `Type` field is not part of the schema, so we have to filter those cases
 # when querying Salesforce.

sync-ago.py

@@ -1,7 +1,6 @@
 import os, sys
 import pytz
 import pandas as pd
-from arcgis import GIS
 from arcgis.features import FeatureLayerCollection
 import datetime
 import cx_Oracle
@@ -10,9 +9,18 @@
 import petl as etl
 import pyproj
 import shapely.wkt
+import citygeo_secrets as cgs
 from shapely.ops import transform as shapely_transformer
 from config import *
 import click
+from utils import *
+
+cgs.set_config(
+    log_level='info',
+    keeper_dir='/scripts/',
+    verify_ssl_certs = False)
+
+AGO_USER = 'AGO/maps.phl.backup'
 
 
 @click.command()
@@ -30,14 +38,7 @@ def sync(day):
                                                f'epsg:{AGO_SRID}',
                                                always_xy=True)
 
-    print('Connecting to AGO...')
-    org = GIS(url='https://phl.maps.arcgis.com',
-                username='maps.phl.data',
-                password=MAPS_PASSWORD,
-                verify_cert=False)
-    print('Connected to AGO.\n')
-
-    flayer = org.content.get(SALESFORCE_AGO_ITEMID)
+    flayer = cgs.connect_with_secrets(get_salesforce_ago_layer, "salesforce API copy", AGO_USER )
     LAYER_OBJECT = flayer.layers[0]
     print(LAYER_OBJECT)
 
@@ -49,21 +50,9 @@ def sync(day):
     else:
         raise AssertionError('Item is not geometric.\n')
 
-
-    def database_connect():
-        user = 'sde'
-        password = DATABRIDGE_SDE_PASSWORD
-        host = DATABRIDGE_HOST
-        service_name = 'GISDBP'
-        port = 1521
-        dsn = cx_Oracle.makedsn(host, port, service_name)
-        # Connect to database
-        db_connect = cx_Oracle.connect(user, password, dsn, encoding="UTF-8")
-        print('Connected to %s' % db_connect)
-        cursor = db_connect.cursor()
-        return cursor
-
-    cursor = database_connect()
+
+    sde_connection = citygeo_secrets.connect_with_secrets(connect_sde, "SDE","databridge-oracle/hostname")
+    cursor = sde_connection.cursor()
     print("Connected to Oracle!\n")
 
 

sync.py

@@ -18,8 +18,10 @@
 import geopetl
 import requests
 from requests.adapters import HTTPAdapter, Retry
+import citygeo_secrets
 from common import *
 from config import *
+from utils import *
 
 #print(os.environ["ORA_TZFILE"])
 
@@ -30,13 +32,13 @@
 
 # if this is set to true in the config
 if TEST:
-    DEST_DB_CONN_STRING = f'{DEST_DB_ACCOUNT}/{THREEONEONE_PASSWORD}@{DEST_TEST_DSN}'
+    #connect to test DB
+    dest_conn = citygeo_secrets.connect_with_secrets(connect_311_test, "GIS_311","databridge-oracle/hostname-testing")
 else:
-    DEST_DB_CONN_STRING = f'{DEST_DB_ACCOUNT}/{THREEONEONE_PASSWORD}@{DEST_PROD_DSN}'
+    # connect to prod DB
+    dest_conn = citygeo_secrets.connect_with_secrets(connect_311, "GIS_311","databridge-oracle/hostname")
+
 
-# Connect to database
-#print("Connecting to oracle, DNS: {}".format(DEST_DB_CONN_STRING)) 
-dest_conn = cx_Oracle.connect(DEST_DB_CONN_STRING)
 dest_conn.autocommit = True
 # 1200000 ms is a 20 minute timeout for really big queries
 # Nvm, 0 for infinite
@@ -59,10 +61,12 @@ def write_log(msg):
 @click.option('--year_refresh', '-y', default=None, help='Retrieve records that were updated in a specific year, then upsert them. Ex: 2017')
 @click.option('--date_column', '-c', default='LastModifiedDate', help='Date column to select cases by from Salesforce. Default is "LastModifiedDate". You can consider using "CreatedDate" when doing full refreshes.')
 def sync(day_refresh, year_refresh, month_refresh, date_column):
-        # Connect to Salesforce
-        sf = Salesforce(username=SF_USER, \
-                        password=SF_PASSWORD, \
-                        security_token=SF_TOKEN)
+        # salesforce_creds = citygeo_secrets.connect_with_secrets(connect_salesforce, "salesforce API copy" )
+        # # Connect to Salesforce
+        # sf = Salesforce(username=salesforce_creds.get('login'), \
+        #                 password=salesforce_creds.get('password'), \
+        #                 security_token=salesforce_creds.get('token'))
+        sf = citygeo_secrets.connect_with_secrets(connect_salesforce_DEV, "salesforce API copy" )
         # supposedly SalesForce() takes a timeout parameter, but it doesn't appear to work.
         # Instead, we can apparently set the tmeout anyway by inserting our own request session
         #session = requests.Session()

utils.py

@@ -0,0 +1,56 @@
+import cx_Oracle
+from simple_salesforce import Salesforce
+import citygeo_secrets
+from arcgis import GIS
+
+# Secret for sync-ago.py
+def get_salesforce_ago_layer(creds:dict): 
+    salesforce_creds = creds['salesforce API copy']
+    ago_creds =  creds["AGO/maps.phl.backup"]
+    org = GIS(url=ago_creds.get('url'),
+                username=ago_creds.get('login'),
+                password=ago_creds.get('password'),
+                verify_cert=False)
+    flayer = org.content.get(salesforce_creds.get('AGO_item_id'))
+    return flayer
+
+# Secret for sync-ago.py
+def connect_sde(creds: dict):
+    db_creds = creds['SDE']
+    db_creds_2 = creds['databridge-oracle/hostname']
+    # Connect to database
+    dsn = cx_Oracle.makedsn(db_creds_2.get('host').get('hostName'), db_creds_2.get('host').get('port'),
+                                   service_name=db_creds_2.get('database'))
+    connection = cx_Oracle.connect(db_creds.get('login'), db_creds.get('password'), dsn,
+                                          encoding="UTF-8")
+    return connection
+
+# Secret for sync.py
+def connect_311(creds: dict):
+    db_creds = creds['GIS_311']
+    db_creds_2 = creds['databridge-oracle/hostname']
+    # Connect to database
+    dsn = cx_Oracle.makedsn(db_creds_2.get('host').get('hostName'), db_creds_2.get('host').get('port'),
+                                   service_name=db_creds_2.get('database'))
+    connection = cx_Oracle.connect(db_creds.get('login'), db_creds.get('password'), dsn,
+                                          encoding="UTF-8")
+    return connection
+
+# Secret for sync.py
+def connect_311_test(creds: dict):
+    db_creds = creds['GIS_311']
+    db_creds_2 = creds['databridge-oracle/hostname-testing']
+    # Connect to database
+    dsn = cx_Oracle.makedsn(db_creds_2.get('host').get('hostName'), db_creds_2.get('host').get('port'),
+                                   service_name=db_creds_2.get('database'))
+    connection = cx_Oracle.connect(db_creds.get('login'), db_creds.get('password'), dsn,
+                                          encoding="UTF-8")
+    return connection
+
+# Secret for sync.py. SHOULD BE RENAMED TO connect_salesforce() AFTER MERGING TO MAIN
+def connect_salesforce_DEV(creds:dict):
+    salesforce_creds=creds['salesforce API copy']
+    sf = Salesforce(username=salesforce_creds.get('login'), \
+                       password=salesforce_creds.get('password'), \
+                       security_token=salesforce_creds.get('token'))
+    return sf