diff --git a/sample_config.py b/sample_config.py index b0d5f7c..59cfd97 100644 --- a/sample_config.py +++ b/sample_config.py @@ -4,19 +4,10 @@ LOG_LEVEL = 'INFO' LOG_PATH = './logs/sync.log' -# Secrets for sync-ago.py -MAPS_PASSWORD = 'password' -SALESFORCE_AGO_ITEMID = '1234abcdf' -DATABRIDGE_SDE_PASSWORD = 'password' -DATABRIDGE_HOST = 'database.host' -DATABRIDGE_DB = 'database' - -THREEONEONE_PASSWORD = 'password' # These describe the destination (enterprise) dataset. # These describe the destination (enterprise) dataset. -DEST_PROD_DSN = '' -DEST_TEST_DSN = '' + DEST_DB_ACCOUNT = '' DEST_TABLE = 'SALESFORCE_CASES' DEST_UPDATED_FIELD = 'updated_datetime' @@ -61,10 +52,7 @@ # Microsoft Teams Web Connector URL MSTEAMS_CONNECTOR = 'https://phila.webhook.office.com/webhookb2/763c9a83-0f38-4eb2-abfc-e0f2f41b6fbb@2046864f-68ea-497d-af34-a6629a6cd700/IncomingWebhook/434dfb8c116d472f8f224cfae367cdc1/2f82d684-85a4-4131-95a3-3342e012faeb' -# These are for querying Salesforce. -SF_USER = '' -SF_PASSWORD = '' -SF_TOKEN = '' + # Most of the filtering for the public view we do in the database, but the # `Type` field is not part of the schema, so we have to filter those cases # when querying Salesforce. diff --git a/sync-ago.py b/sync-ago.py index 37606f4..0154b75 100644 --- a/sync-ago.py +++ b/sync-ago.py @@ -1,7 +1,6 @@ import os, sys import pytz import pandas as pd -from arcgis import GIS from arcgis.features import FeatureLayerCollection import datetime import cx_Oracle @@ -10,9 +9,18 @@ import petl as etl import pyproj import shapely.wkt +import citygeo_secrets as cgs from shapely.ops import transform as shapely_transformer from config import * import click +from utils import * + +cgs.set_config( + log_level='info', + keeper_dir='/scripts/', + verify_ssl_certs = False) + +AGO_USER = 'AGO/maps.phl.backup' @click.command() @@ -30,14 +38,7 @@ def sync(day): f'epsg:{AGO_SRID}', always_xy=True) - print('Connecting to AGO...') - org = GIS(url='https://phl.maps.arcgis.com', - username='maps.phl.data', - password=MAPS_PASSWORD, - verify_cert=False) - print('Connected to AGO.\n') - - flayer = org.content.get(SALESFORCE_AGO_ITEMID) + flayer = cgs.connect_with_secrets(get_salesforce_ago_layer, "salesforce API copy", AGO_USER ) LAYER_OBJECT = flayer.layers[0] print(LAYER_OBJECT) @@ -49,21 +50,9 @@ def sync(day): else: raise AssertionError('Item is not geometric.\n') - - def database_connect(): - user = 'sde' - password = DATABRIDGE_SDE_PASSWORD - host = DATABRIDGE_HOST - service_name = 'GISDBP' - port = 1521 - dsn = cx_Oracle.makedsn(host, port, service_name) - # Connect to database - db_connect = cx_Oracle.connect(user, password, dsn, encoding="UTF-8") - print('Connected to %s' % db_connect) - cursor = db_connect.cursor() - return cursor - - cursor = database_connect() + + sde_connection = citygeo_secrets.connect_with_secrets(connect_sde, "SDE","databridge-oracle/hostname") + cursor = sde_connection.cursor() print("Connected to Oracle!\n") diff --git a/sync.py b/sync.py index 99bb4e1..b017fae 100644 --- a/sync.py +++ b/sync.py @@ -18,8 +18,10 @@ import geopetl import requests from requests.adapters import HTTPAdapter, Retry +import citygeo_secrets from common import * from config import * +from utils import * #print(os.environ["ORA_TZFILE"]) @@ -30,13 +32,13 @@ # if this is set to true in the config if TEST: - DEST_DB_CONN_STRING = f'{DEST_DB_ACCOUNT}/{THREEONEONE_PASSWORD}@{DEST_TEST_DSN}' + #connect to test DB + dest_conn = citygeo_secrets.connect_with_secrets(connect_311_test, "GIS_311","databridge-oracle/hostname-testing") else: - DEST_DB_CONN_STRING = f'{DEST_DB_ACCOUNT}/{THREEONEONE_PASSWORD}@{DEST_PROD_DSN}' + # connect to prod DB + dest_conn = citygeo_secrets.connect_with_secrets(connect_311, "GIS_311","databridge-oracle/hostname") + -# Connect to database -#print("Connecting to oracle, DNS: {}".format(DEST_DB_CONN_STRING)) -dest_conn = cx_Oracle.connect(DEST_DB_CONN_STRING) dest_conn.autocommit = True # 1200000 ms is a 20 minute timeout for really big queries # Nvm, 0 for infinite @@ -59,10 +61,12 @@ def write_log(msg): @click.option('--year_refresh', '-y', default=None, help='Retrieve records that were updated in a specific year, then upsert them. Ex: 2017') @click.option('--date_column', '-c', default='LastModifiedDate', help='Date column to select cases by from Salesforce. Default is "LastModifiedDate". You can consider using "CreatedDate" when doing full refreshes.') def sync(day_refresh, year_refresh, month_refresh, date_column): - # Connect to Salesforce - sf = Salesforce(username=SF_USER, \ - password=SF_PASSWORD, \ - security_token=SF_TOKEN) + # salesforce_creds = citygeo_secrets.connect_with_secrets(connect_salesforce, "salesforce API copy" ) + # # Connect to Salesforce + # sf = Salesforce(username=salesforce_creds.get('login'), \ + # password=salesforce_creds.get('password'), \ + # security_token=salesforce_creds.get('token')) + sf = citygeo_secrets.connect_with_secrets(connect_salesforce_DEV, "salesforce API copy" ) # supposedly SalesForce() takes a timeout parameter, but it doesn't appear to work. # Instead, we can apparently set the tmeout anyway by inserting our own request session #session = requests.Session() diff --git a/utils.py b/utils.py new file mode 100644 index 0000000..38c8349 --- /dev/null +++ b/utils.py @@ -0,0 +1,56 @@ +import cx_Oracle +from simple_salesforce import Salesforce +import citygeo_secrets +from arcgis import GIS + +# Secret for sync-ago.py +def get_salesforce_ago_layer(creds:dict): + salesforce_creds = creds['salesforce API copy'] + ago_creds = creds["AGO/maps.phl.backup"] + org = GIS(url=ago_creds.get('url'), + username=ago_creds.get('login'), + password=ago_creds.get('password'), + verify_cert=False) + flayer = org.content.get(salesforce_creds.get('AGO_item_id')) + return flayer + +# Secret for sync-ago.py +def connect_sde(creds: dict): + db_creds = creds['SDE'] + db_creds_2 = creds['databridge-oracle/hostname'] + # Connect to database + dsn = cx_Oracle.makedsn(db_creds_2.get('host').get('hostName'), db_creds_2.get('host').get('port'), + service_name=db_creds_2.get('database')) + connection = cx_Oracle.connect(db_creds.get('login'), db_creds.get('password'), dsn, + encoding="UTF-8") + return connection + +# Secret for sync.py +def connect_311(creds: dict): + db_creds = creds['GIS_311'] + db_creds_2 = creds['databridge-oracle/hostname'] + # Connect to database + dsn = cx_Oracle.makedsn(db_creds_2.get('host').get('hostName'), db_creds_2.get('host').get('port'), + service_name=db_creds_2.get('database')) + connection = cx_Oracle.connect(db_creds.get('login'), db_creds.get('password'), dsn, + encoding="UTF-8") + return connection + +# Secret for sync.py +def connect_311_test(creds: dict): + db_creds = creds['GIS_311'] + db_creds_2 = creds['databridge-oracle/hostname-testing'] + # Connect to database + dsn = cx_Oracle.makedsn(db_creds_2.get('host').get('hostName'), db_creds_2.get('host').get('port'), + service_name=db_creds_2.get('database')) + connection = cx_Oracle.connect(db_creds.get('login'), db_creds.get('password'), dsn, + encoding="UTF-8") + return connection + +# Secret for sync.py. SHOULD BE RENAMED TO connect_salesforce() AFTER MERGING TO MAIN +def connect_salesforce_DEV(creds:dict): + salesforce_creds=creds['salesforce API copy'] + sf = Salesforce(username=salesforce_creds.get('login'), \ + password=salesforce_creds.get('password'), \ + security_token=salesforce_creds.get('token')) + return sf