From 3486229f30e98de00b221ed141c673ec00065e45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20Mu=C3=B1oz=20C=C3=A1rdenas?= Date: Tue, 27 Mar 2018 13:18:51 +0200 Subject: [PATCH] Delete scripts --- scripts/__init__.py | 0 scripts/char_to_int.py | 103 --------------------- scripts/check_protection.py | 30 ------- scripts/database_test.py | 27 ------ scripts/fill_database.py | 96 -------------------- scripts/fill_neo4j.py | 102 --------------------- scripts/getData.py | 162 --------------------------------- scripts/import_adminlevels.py | 127 -------------------------- scripts/import_sectors.py | 106 ---------------------- scripts/migrate_user.py | 42 --------- scripts/neo4j_test.py | 31 ------- scripts/sync_tables.py | 164 ---------------------------------- scripts/update_checklist.py | 52 ----------- scripts/workflow_migration.py | 70 --------------- 14 files changed, 1112 deletions(-) delete mode 100755 scripts/__init__.py delete mode 100755 scripts/char_to_int.py delete mode 100644 scripts/check_protection.py delete mode 100644 scripts/database_test.py delete mode 100644 scripts/fill_database.py delete mode 100644 scripts/fill_neo4j.py delete mode 100755 scripts/getData.py delete mode 100755 scripts/import_adminlevels.py delete mode 100755 scripts/import_sectors.py delete mode 100755 scripts/migrate_user.py delete mode 100644 scripts/neo4j_test.py delete mode 100755 scripts/sync_tables.py delete mode 100755 scripts/update_checklist.py delete mode 100644 scripts/workflow_migration.py diff --git a/scripts/__init__.py b/scripts/__init__.py deleted file mode 100755 index e69de29bb..000000000 diff --git a/scripts/char_to_int.py b/scripts/char_to_int.py deleted file mode 100755 index 1e099a0ff..000000000 --- a/scripts/char_to_int.py +++ /dev/null @@ -1,103 +0,0 @@ -""" -import json data from API -IMPORTANT!! you must turn off pagination for this to work from a URL and get all -country records -Install module django-extensions -Runs twice via function calls at bottom once -""" -from django.db import connection, transaction - -cursor = connection.cursor() -from os.path import exists -import csv -import unicodedata -import sys -import urllib2 -from datetime import date -from workflow.models import ProjectAgreement, ProjectComplete -import re - -def run(): - print "Running Script..." - - trim_list = ['estimated_budget', 'actual_cost','total_cost','agency_cost','local_total_cost','local_agency_cost'] - - #get all the projects and loop over them - getProjects = ProjectComplete.objects.all() - for item in getProjects: - if item.total_cost and item.local_total_cost != "Nil": - print item.total_cost - trim_item = re.sub("[^0-9.]", "", item.total_cost) - trim_item = float(trim_item) - print trim_item - ProjectComplete.objects.all().filter(id=item.id).update(total_cost=trim_item) - elif item.total_cost in [None, '','Nil']: - ProjectComplete.objects.all().filter(id=item.id).update(total_cost=0.00) - - if item.estimated_budget and item.local_total_cost != "Nil": - print item.estimated_budget - trim_item = re.sub("[^0-9.]", "", item.estimated_budget) - trim_item = float(trim_item) - print trim_item - ProjectComplete.objects.all().filter(id=item.id).update(estimated_budget=trim_item) - elif item.estimated_budget in [None, '','Nil']: - ProjectComplete.objects.all().filter(id=item.id).update(estimated_budget=0.00) - - if item.agency_cost and item.local_total_cost != "Nil": - print item.agency_cost - trim_item = re.sub("[^0-9.]", "", item.agency_cost) - trim_item = float(trim_item) - print trim_item - ProjectComplete.objects.all().filter(id=item.id).update(agency_cost=trim_item) - elif item.agency_cost in [None, '','Nil']: - ProjectComplete.objects.all().filter(id=item.id).update(agency_cost=0.00) - - if item.local_total_cost and item.local_total_cost != "Nil": - print item.local_total_cost - trim_item = re.sub("[^0-9.]", "", item.local_total_cost) - trim_item = float(trim_item) - print trim_item - ProjectComplete.objects.all().filter(id=item.id).update(local_total_cost=trim_item) - elif item.local_total_cost in [None, '','Nil']: - ProjectComplete.objects.all().filter(id=item.id).update(local_total_cost=0.00) - - if item.local_agency_cost and item.local_agency_cost != "Nil": - print item.local_agency_cost - trim_item = re.sub("[^0-9.]", "", item.local_agency_cost) - trim_item = float(trim_item) - print trim_item - ProjectComplete.objects.all().filter(id=item.id).update(local_agency_cost=trim_item) - elif item.local_agency_cost in [None, '','Nil']: - ProjectComplete.objects.all().filter(id=item.id).update(local_agency_cost=0.00) - - trim_list = ['total_estimated_budget','local_total_estimated_budget','local_mc_estimated_budget'] - - #get all the projects and loop over them - getProjectsComplete = ProjectAgreement.objects.all() - for item in getProjectsComplete: - if item.total_estimated_budget and item.total_estimated_budget != "Nil": - print item.total_estimated_budget - trim_item = re.sub("[^0-9.]", "", item.total_estimated_budget) - trim_item = float(trim_item) - print trim_item - ProjectAgreement.objects.all().filter(id=item.id).update(total_estimated_budget=trim_item) - elif item.total_estimated_budget in [None, '','Nil']: - ProjectAgreement.objects.all().filter(id=item.id).update(total_estimated_budget=0.00) - - if item.local_total_estimated_budget and item.local_total_estimated_budget != "Nil": - print item.local_total_estimated_budget - trim_item = re.sub("[^0-9.]", "", item.local_total_estimated_budget) - trim_item = float(trim_item) - print trim_item - ProjectAgreement.objects.all().filter(id=item.id).update(local_total_estimated_budget=trim_item) - elif item.local_total_estimated_budget in [None, '','Nil']: - ProjectAgreement.objects.all().filter(id=item.id).update(local_total_estimated_budget=0.00) - - if item.local_mc_estimated_budget and item.local_mc_estimated_budget != "Nil": - print item.local_mc_estimated_budget - trim_item = re.sub("[^0-9.]", "", item.local_mc_estimated_budget) - trim_item = float(trim_item) - print trim_item - ProjectAgreement.objects.all().filter(id=item.id).update(local_mc_estimated_budget=trim_item) - elif item.local_mc_estimated_budget in [None, '','Nil']: - ProjectAgreement.objects.all().filter(id=item.id).update(local_mc_estimated_budget=0.00) \ No newline at end of file diff --git a/scripts/check_protection.py b/scripts/check_protection.py deleted file mode 100644 index 4c69b5f50..000000000 --- a/scripts/check_protection.py +++ /dev/null @@ -1,30 +0,0 @@ -from workflow.models import * -import urllib -import os -import re - -def run(): - print "Checking security status on models" - try: - check_status() - except Exception, e: - print(e.message) - - -def check_status(): - model_files = [] - for dir in os.listdir("./"): - p = "./" + dir - if os.path.exists(p) and os.path.isdir(p) and os.path.exists(p + "/models.py"): - print dir - with open(p + "/models.py", "r") as mf: - fc = "" - for l in mf.readlines(): - fc = fc + l - - search = re.findall("class (.*)\((.*)\)", fc) - - if search is not None: - for res in search: - if res[1] is not "SecurityModel": - print " "+res[0] + " " + res[1] diff --git a/scripts/database_test.py b/scripts/database_test.py deleted file mode 100644 index e3f8b02ea..000000000 --- a/scripts/database_test.py +++ /dev/null @@ -1,27 +0,0 @@ -import sys -import time - -from workflow.models import WorkflowLevel1 - - -def run(): - print("Starting Database test") - n = 1000 - size = 0 - t1 = time.time() - - for i in range(0,n): - result = simple_query() - #result = relation_query() - size = size + sys.getsizeof(result) - - t2 = time.time() - - print("Executed %d queries in %4.2fs time receiving %8.4f KiloBytes of data" % (n, (t2-t1), size/1024)) - - -def simple_query(): - objects = WorkflowLevel1.objects.all() - #for o in objects: - # print o.id - return objects \ No newline at end of file diff --git a/scripts/fill_database.py b/scripts/fill_database.py deleted file mode 100644 index 9f509920f..000000000 --- a/scripts/fill_database.py +++ /dev/null @@ -1,96 +0,0 @@ -import random -import traceback - -from model_mommy import mommy - -from indicators.models import Indicator, IndicatorType -from workflow.models import * - - -def run(): - n = 1000 - try: - for i in range(0,n): - create_workflowmodels() - #create_indicators() - except Exception, e: - print(e.message) - print(traceback.format_exc()) - - -def create_indicators(): - it = mommy.make(IndicatorType, _fill_optional=True) - wf1 = mommy.make(WorkflowLevel1, _fill_optional=True) - i = mommy.make(Indicator, - make_m2m=True, - _quantity=1, - _fill_optional=True, - indicator_type=[it], - workflowlevel1=[wf1]) - - -def create_workflowmodels(): - user = TolaUser.objects.get(employee_number=123) - - c = mommy.make(Country, - make_m2m=True, - _quantity=1, - _fill_optional=True, - country="Country %d"% (random.randint(1, 100))) - - fund_code = mommy.make(FundCode, - make_m2m=True, - _quantity=1, - _fill_optional=True) - - sector = mommy.make(Sector, - make_m2m=True, - _quantity=1, - _fill_optional=True) - - sub_sector = mommy.make(Sector, - make_m2m=True, - _quantity=1, - _fill_optional=True) - - wf1 = mommy.make(WorkflowLevel1, - make_m2m=True, - _quantity=1, - _fill_optional=True, - country=c, - fund_code=fund_code, - sector=sector, - sub_sector=sub_sector, - user_access=[user]) - - ptype = mommy.make(ProjectType, - make_m2m=True, - _quantity=1, - _fill_optional=True) - - site = mommy.make(SiteProfile, - make_m2m=True, - _quantity=1, - _fill_optional=True, - approved_by=user, - filled_by=user, - location_verified_by=user) - - """stakeholder = mommy.make(Stakeholder, - make_m2m=True, - _quantity=1, - _fill_optional=True, - country=c, - sector=sector, - )""" - - wf2 = mommy.make(WorkflowLevel2, - make_m2m=True, - _quantity=1, - _fill_optional=True, - workflowlevel1=wf1[0], - project_type=ptype[0], - site=site, - sector=sector[0], - #stakeholder=stakeholder - ) \ No newline at end of file diff --git a/scripts/fill_neo4j.py b/scripts/fill_neo4j.py deleted file mode 100644 index 065814053..000000000 --- a/scripts/fill_neo4j.py +++ /dev/null @@ -1,102 +0,0 @@ -from os import listdir -from os.path import isfile, join - -from neo4j.v1 import GraphDatabase, basic_auth -import pickle, os, random, string - - -def generate_text(length): - return "".join([random.choice(string.letters) for i in xrange(length)]) - -def create_countries(): - countries = ["Germany", "Afghanistan", "USA", "India", "Nigeria", "Brazil"] - count = 0 - for c in countries: - count = count + 1 - - session.run( - "MERGE (c:Country {name:{name}, id:{id}})", - { - "id": count, - "name": c - } - ) - -def create_indicators(max=100): - for i in range(0, max): - session.run( - "MERGE (i:Indicator {name:{name}, type:{type}, description:{description}, status:{status}})", - { - "id": i, - "name": generate_text(random.randint(5,20)), - "type": generate_text(random.randint(5,10)), - "description": generate_text(random.randint(10,20)), - "status": generate_text(random.randint(5,10)) - } - ) - -def create_demo(max_projects): - for i in range(0, max_projects): - # Add workflow object - - session.run( - "MERGE (w:Workflow {name:{name}, project_activity:{project_activity}, description:{description}, funding_status:{funding_status}})", - { - "id": i, - "name": generate_text(random.randint(5,20)), - "description": generate_text(random.randint(50,500)), - "project_activity": generate_text(random.randint(10,20)), - "funding_status": generate_text(random.randint(5,10)) - } - ) - - - - -""" - country - gbdTx = "MATCH (c:Condition {title: {cond_title}}) " \ - "MATCH (b:Burden) WHERE {icd10code} IN b.icd10codes " \ - "CREATE UNIQUE (c)-[:HAS_BURDEN]->(b)" - - stakeholder - - Sector - - Indicator -""" -def create_relations(): - result = session.run("MATCH (w:Workflow) RETURN w.name as name, ID(w) as id") - - for record in result: - print(record["name"], record["id"]) - - session.run("MATCH (w:Workflow) WHERE ID(w) = {w_id} MATCH (c:Country) WHERE ID(c) = {c_id} CREATE UNIQUE (w)-[:LOCATED_IN]->(c)", - {"w_id": record["id"], "c_id": random.randint(0, 6)} - ) - for n in range(1,random.randint(1,5)): - session.run( - "MATCH (w:Workflow) WHERE ID(w) = {w_id} MATCH (i:Indicator) WHERE ID(i) = {i_id} CREATE (w)-[:USES]->(i)", - {"w_id": record["id"], "i_id": random.randint(0, 100)} - ) - - """ - - - session.run( - "MATCH (w:Workflow {id: {w_id}}), (c:Country {id:{c_id}}), (i:Indicator {id:{i_id}})" + - "CREATE UNIQUE (w)-[:LOCATED_IN]->(c)" + - "CREATE UNIQUE (w)-[:USES]->(i)", - {"w_id": i, "c_id": random.randint(0, 100), "i_id": random.randint(0, 6)} -)""" - -# establish db connection -driver = GraphDatabase.driver("bolt://neo4j-test.tola.io:7687", auth=basic_auth("neo4j", "")) -session = driver.session() - -create_countries() -create_indicators(100) -print("creating demo files") -create_demo(1000) -print("creating demo relations") -create_relations() diff --git a/scripts/getData.py b/scripts/getData.py deleted file mode 100755 index 39281210a..000000000 --- a/scripts/getData.py +++ /dev/null @@ -1,162 +0,0 @@ -""" -import json data from API -IMPORTANT!! you must turn off pagination for this to work from a URL and get all -country records -Install module django-extensions -Runs twice via function calls at bottom once -""" -from django.db import connection, transaction - -cursor = connection.cursor() -from os.path import exists -import json -import unicodedata -import sys -import urllib2 -from datetime import date -from workflow.models import Country, WorkflowLevel1 -import urllib - -def run(): - print "Uploading JSON data" - -type = "Program" -program_country = 1 - - -def getAllData(url, type, program_country): - - # set url for json feed here - json_file = urllib2.urlopen(url) - - #load data - data = json.load(json_file) - json_file.close() - - #print data - - #query to mysql database after parsing json data - def saveCountries(keys_to_sql, vars_to_sql): - #save the original keys list for update in case we need to run that - save_keys = keys_to_sql - keys_to_sql = ", ".join(map(str, keys_to_sql)) - - query = "INSERT INTO activitydb_country (country,code) VALUES ('%s','%s')" % (vars_to_sql[0], vars_to_sql[1]) - print query - - try: - cursor.execute(query) - transaction.commit() - except Exception, err: - sys.stderr.write('ERROR: %s\n' % str(err)) - column = save_keys[1] - value = 1 - country = vars_to_sql[0] - if type == "country": - query_update = "UPDATE activitydb_country set country = %s where lower(%(type)s) = '%s'" % ( - column, value, country.lower()) - try: - cursor.execute(query_update) - transaction.commit() - except Exception, err: - sys.stderr.write('ERROR: %s\n' % str(err)) - return 1 - pass - - #query to mysql database after parsing json data - def savePrograms(keys_to_sql, vars_to_sql,program_country): - #save the original keys list for update in case we need to run that - save_keys = keys_to_sql - keys_to_sql = ", ".join(map(str, keys_to_sql)) - - var_to_tuple = tuple(vars_to_sql) - print var_to_tuple - - query = "INSERT INTO activitydb_program (%s) VALUES %s" % (keys_to_sql, var_to_tuple) - print query - - try: - cursor.execute(query) - transaction.commit() - except Exception, err: - sys.stderr.write('ERROR: %s\n' % str(err)) - pass - - latest = WorkflowLevel1.objects.latest('id') - - query2 = "INSERT INTO activitydb_program_country (country_id,program_id) VALUES (%s,%s)" % (program_country, latest.id) - - print query2 - try: - cursor.execute(query2) - transaction.commit() - except Exception, err: - sys.stderr.write('ERROR: %s\n' % str(err)) - pass - - - for row in data: - print row - vars_to_sql = [] - keys_to_sql = [] - for new_key, new_value in row.iteritems(): - try: - new_key = new_key.encode('ascii','ignore') - new_value = new_value.encode('ascii','ignore') - except Exception, err: - sys.stderr.write('ERROR: %s\n' % str(err)) - #print new_key - #print new_value - if type == "Country": - if new_value: - #country or region related columns only - if new_key in ('country','region','iso_code'): - #change iso_code to code for DB table - if new_key == 'iso_code': - new_key = 'code' - keys_to_sql.append(new_key) - vars_to_sql.append(new_value) - elif type == "Program": - if new_value: - #country or region related columns only - if new_key in ('gaitid','funding_status','granttitle','cost_center'): - #change iso_code to code for DB table - if new_key == 'granttitle': - new_key = 'name' - keys_to_sql.append(new_key) - vars_to_sql.append(new_value) - - #add create_date to comma seperated list of columns - keys_to_sql.append("create_date") - #append todays date to var - today = date.today() - today.strftime('%Y-%m-%d') - today = str(today) - vars_to_sql.append(today) - - - #add description to comma separated list of columns - keys_to_sql.append("description") - #append todays date to var - vars_to_sql.append("") - - if type == "Country": - saveCountries(keys_to_sql, vars_to_sql) - elif type == "Program": - savePrograms(keys_to_sql, vars_to_sql, program_country) - -# get an updated json data file for the hub and update or insert new records -#print "Country" -#getAllData("https://mcapi.mercycorps.org/authoritativecountry/?gait=True&format=json", "Country") - -#get an updated json data file for the hub and update or insert new records -print "Program" -getCountries = Country.objects.all() -for country in getCountries: - print country.country - safe_country = urllib.quote_plus(country.country) - program_url = "http://mcapi.mercycorps.org/gaitprogram/?country=%s&format=json" % (safe_country) - print program_url - getAllData(program_url, "Program", int(country.id)) - -print "Alright, all done." diff --git a/scripts/import_adminlevels.py b/scripts/import_adminlevels.py deleted file mode 100755 index e445e7170..000000000 --- a/scripts/import_adminlevels.py +++ /dev/null @@ -1,127 +0,0 @@ -""" -import json data from API -IMPORTANT!! you must turn off pagination for this to work from a URL and get all -country records -Install module django-extensions -Runs twice via function calls at bottom once -Syntax: sudo py manage.py runscript import_adminlevels -""" -from django.db import connection - -cursor = connection.cursor() -import csv -from workflow.models import Country, Province, District, AdminLevelThree, Village - -def run(): - print "Uploading Country Admin data" - - -def getAllData(getCountry,file_name): - - with open(file_name, 'rb') as csvfile: - country = csv.reader(csvfile, delimiter=',', quotechar='"') - print "PROVINCE (LEVEL 1) !!!!!!" - #check for province and add new ones - for row in country: - column_num = 0 - for column in row: - if column_num == 1: - print "Province=" - print column - try: - Province.objects.get(name=column, country=getCountry) - except Province.DoesNotExist: - new_prov = Province(name=column, country=getCountry) - new_prov.save() - print column - column_num = column_num + 1 - - with open(file_name, 'rb') as csvfile2: - country2 = csv.reader(csvfile2, delimiter=',', quotechar='"') - #check for distrcit and add new one - for row in country2: - print "DISTRICTS (LEVEL 2) !!!!!!" - column_num = 0 - for column in row: - if column_num == 1: - getProvince = Province.objects.get(name=column, country=getCountry) - - if column_num == 2: - print "District=" - print column - - try: - District.objects.get(name=column, province=getProvince) - except District.DoesNotExist: - new_district = District(name=column, province=getProvince) - new_district.save() - - column_num = column_num + 1 - - with open(file_name, 'rb') as csvfile2: - country2 = csv.reader(csvfile2, delimiter=',', quotechar='"') - #check for level3 and add new one - for row in country2: - print "LEVEL 3 !!!!!!" - column_num = 0 - for column in row: - if column_num == 1: - getProvince = Province.objects.get(name=column, country=getCountry) - - if column_num == 2: - getDistrict = District.objects.get(name=column, province=getProvince) - - if column_num == 3: - print "AdminLevelThree=" - print column - - try: - AdminLevelThree.objects.get(name=column, district=getDistrict) - except AdminLevelThree.DoesNotExist: - new_level_3 = AdminLevelThree(name=column, district=getDistrict) - new_level_3.save() - - column_num = column_num + 1 - - with open(file_name, 'rb') as csvfile2: - country2 = csv.reader(csvfile2, delimiter=',', quotechar='"') - #check for village and add new one - for row in country2: - print "VILLAGE !!!!!" - column_num = 0 - for column in row: - if column_num == 1: - getProvince = Province.objects.get(name=column, country=getCountry) - - if column_num == 2: - getDistrict = District.objects.get(name=column, province=getProvince) - - if column_num == 3: - getAdminLevel3 = AdminLevelThree.objects.get(name=column, district=getDistrict) - - if column_num == 4: - print "Village=" - print column - - try: - Village.objects.get(name=column, admin_3=getAdminLevel3) - except Village.DoesNotExist: - new_level_3 = Village(name=column, admin_3=getAdminLevel3) - new_level_3.save() - - column_num = column_num + 1 - - -# UNCOMMENT AND UPDATE TO IMPORT -print "IMPORTING South Sudan !!!!!!" -getCountry, created = Country.objects.get_or_create(country="Liberia") -file_name = "fixtures/Liberia_Admin.csv" -getAllData(getCountry, file_name) - -""" -print "IMPORTING Uganda !!!!!!" -getCountry, created = Country.objects.get_or_create(country="Uganda") -file_name = "fixtures/admin-uganda.csv" -getAllData(getCountry, file_name) - -""" diff --git a/scripts/import_sectors.py b/scripts/import_sectors.py deleted file mode 100755 index db6b25af3..000000000 --- a/scripts/import_sectors.py +++ /dev/null @@ -1,106 +0,0 @@ -""" -import json data from API -IMPORTANT!! you must turn off pagination for this to work from a URL and get all -country records -Install module django-extensions -Runs twice via function calls at bottom once -Syntax: sudo py manage.py runscript import_adminlevels -""" -from django.db import connection - -cursor = connection.cursor() -import csv -from workflow.models import Sector, SectorRelated, Organization - -def run(): - print "Uploading Sectors" - - -def getAllData(file_name): - org = Organization.objects.get(name="TolaData") - - with open(file_name, 'rb') as csvfile: - sector = csv.reader(csvfile, delimiter=',', quotechar='"') - print "Sector" - #check for province and add new ones - for row in sector: - column_num = 0 - for column in row: - if column_num == 1: - print "Sector=" - print column - try: - Sector.objects.get(sector=column, organization=org) - except Sector.DoesNotExist: - new_sector = Sector(sector=column, organization=org) - new_sector.save() - print column - if column_num == 2: - print "Sector=" - print column - try: - Sector.objects.get(sector=column) - except Sector.DoesNotExist: - new_sector = Sector(sector=column, organization=org) - new_sector.save() - print column - column_num = column_num + 1 - - with open(file_name, 'rb') as csvfile2: - sector2 = csv.reader(csvfile2, delimiter=',', quotechar='"') - #check for distrcit and add new one - delete = Sector.objects.all().delete() - delete = SectorRelated.objects.all().delete() - org = Organization.objects.get(name="TolaData") - - for row in sector2: - - sector = "" - sector_related = "" - column_num = 0 - for column in row: - - if column_num == 0: - sector = column - - if column_num == 1: - sector_related = column - column_num = column_num + 1 - - # Add the sector to the sector object - try: - getSector = Sector.objects.get(sector=sector) - except Sector.DoesNotExist: - new_sector = Sector(sector=sector, organization=org) - new_sector.save() - getSector = Sector.objects.get(sector=sector) - - # add the sub-sector to the sector object - print sector_related - try: - getSubSector = Sector.objects.get(sector=sector_related) - except Sector.DoesNotExist: - new_sector = Sector(sector=sector_related, organization=org) - new_sector.save() - getSubSector = Sector.objects.get(sector=sector_related) - - # add sector and sub-sector to the related object - try: - checksub = SectorRelated.objects.filter(sector=getSector, sector_related=getSubSector, organization=org).get() - except SectorRelated.DoesNotExist: - print "adding new sector related" - print getSector - print getSubSector - print "or not" - new_related = SectorRelated(sector=getSector, sector_related=getSubSector, organization=org) - new_related.save() - print row - - - -# UNCOMMENT AND UPDATE TO IMPORT -print "IMPORTING Sectors" -file_name = "fixtures/Sector-SubSector.csv" -getAllData(file_name) - - diff --git a/scripts/migrate_user.py b/scripts/migrate_user.py deleted file mode 100755 index b6ad84b1f..000000000 --- a/scripts/migrate_user.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -import json data from API -IMPORTANT!! you must turn off pagination for this to work from a URL and get all -country records -Install module django-extensions -Runs twice via function calls at bottom once -""" - -from djangocosign.models import UserProfile, Country as CosignCountry -from workflow.models import TolaUser, Country - - -def run(): - print "Migrating User from Djangocosign to TolaUser" - - -def getAllData(): - getUsers = UserProfile.objects.all() - for item in getUsers: - cosign_user = UserProfile.objects.get(user=item.user) - try: - get_user = TolaUser.objects.get(user=item.user) - except TolaUser.DoesNotExist: - get_user = None - print cosign_user.user - try: - get_country = Country.objects.get(code=cosign_user.country.iso_two_letters_code) - except Country.DoesNotExist: - get_country = None - if get_user: - print "user exists" - else: - get_user = TolaUser.objects.create( - title=cosign_user.title, - name=cosign_user.name, - user=cosign_user.user, - modified_by=cosign_user.modified_by, - country=get_country - ) - get_user.save() - -getAllData() \ No newline at end of file diff --git a/scripts/neo4j_test.py b/scripts/neo4j_test.py deleted file mode 100644 index 52e688b72..000000000 --- a/scripts/neo4j_test.py +++ /dev/null @@ -1,31 +0,0 @@ -from neo4j.v1 import GraphDatabase, basic_auth -import time, sys - -driver = GraphDatabase.driver("bolt://neo4j-test.tola.io:7687", auth=basic_auth("neo4j", "")) -session = driver.session() - - -def simple_query(): - res = session.run("MATCH (w:Workflow) RETURN * LIMIT 2000") # w.name as name, ID(w) as id - - #for record in res: - #print(record["id"]) - - return res - - -def relation_query(): - return session.run('MATCH (w:Workflow)-[r:USES]->(i:Indicator), (w)-[l:LOCATED_IN]->(c1:Country {name:"Afghanistan"}) RETURN *') - -n = 1000 -size = 0 -t1 = time.time() - -for i in range(0,n): - #result = simple_query() - result = relation_query() - size = size + sys.getsizeof(result) - -t2 = time.time() - -print("Executed %d queries in %4.2fs time receiving %8.4f KiloBytes of data" % (n, (t2-t1), size/1024)) diff --git a/scripts/sync_tables.py b/scripts/sync_tables.py deleted file mode 100755 index a421fce60..000000000 --- a/scripts/sync_tables.py +++ /dev/null @@ -1,164 +0,0 @@ -""" -import json data from API -IMPORTANT!! you must turn off pagination for this to work from a URL and get all -country records -Install module django-extensions -Runs twice via function calls at bottom once -""" -import json -import requests - -from workflow.models import Country, TolaUser, TolaSites, Organization, WorkflowLevel1, WorkflowLevel2 - - -def _get_headers(): - """ - Authentication for Tables - Get the header information to send to tables in the request - :return: headers - """ - tables = TolaSites.objects.get(site_id=1) - if tables.tola_tables_token: - headers = {'content-type': 'application/json', - 'Authorization': 'Token ' + tables.tola_tables_token} - else: - headers = {'content-type': 'application/json'} - print "Token Not Found" - - return headers - - -def _send_to_tables(section,payload, id): - """ - send the requests to individual API endpoint in tables - with the payload and id - :param section: model class on tables and activity - :return: status: success of failure of request - """ - headers = _get_headers() - - # get the table URL - tables = TolaSites.objects.get(site_id=1) - - print "COMMENT POST TO TABLES" - post_url = tables.tola_tables_url + "api/" + section + "/" + str(id) - print post_url - resp = requests.post(post_url, json=payload, headers=headers, verify=False) - status = resp.reason + " : " + str(resp.status_code) - print status - - if str(resp.status_code) == "405": - print "COMMENT 405 means it likely does not exist so create a new one" - post_url = tables.tola_tables_url + "api/" + section - print post_url - resp = requests.post(post_url, json=payload, headers=headers, verify=False) - status = resp.reason - print resp.content - - if str(resp.status_code) == "200": - status = "success" - - return status - - -def _check_org(org): - """ - Get or create the org in Tables - :param org: - :return: the org URL in tables - """ - status = None - headers = _get_headers() - - # get the table URL - tables = TolaSites.objects.get(site_id=1) - check_org_url = tables.tola_tables_url + "api/organization?format=json&name=" + org - check_resp = requests.get(check_org_url, headers=headers, verify=False) - - print check_resp.content - data = json.loads(check_resp.content) - - if not data: - # print "COMMENT REPOST TO CREATE ORG" - post_org_url = tables.tola_tables_url + "api/organization" - payload = {"name": org} - post_resp = requests.post(post_org_url, json=payload, headers=headers, verify=False) - org_url = post_org_url - - status = "post_resp=" + str(post_resp.status_code) + " " + post_org_url - - print status - # print "COMMENT RECHECK ORG" - check_org_url = tables.tola_tables_url + "api/organization?format=json&name=" + org - check_resp = requests.get(check_org_url, headers=headers, verify=False) - - status = "check_org=" + str(check_resp.status_code) + " " + org_url - # get data response from post - data = json.loads(check_resp.content) - - print data[0] - org_id = str(data[0]['id']) - - print status - - return org_id - - -def update_level1(): - """ - Update or create each workflowlevel1 in tables - :return: - """ - # update TolaTables with program data - Level1 = WorkflowLevel1.objects.all() - - # each level1 send to tables - for item in Level1: - print item.countries - # check to see if the organization exists on tables first if not it check_org will create it - get_org = Organization.objects.all().get(country__country=item.countries) - org_id = _check_org(get_org.name) - # set payload and deliver - print org_id - payload = {'level1_uuid': item.level1_uuid, 'name': item.name, 'organization': org_id} - print payload - send = _send_to_tables(section="workflowlevel1", payload=payload, id=item.id) - - print send - - -def update_level2(): - """ - Update or create each workflowlevel1 in tables - :return: - """ - # update TolaTables with program data - Level2 = WorkflowLevel2.objects.all() - - # each level1 send to tables - for item in Level2: - print item.workflowlevel1.countries - # check to see if the organization exists on tables first if not it check_org will create it - get_org = Organization.objects.all().get(country__country=item.workflowlevel1.countries) - org_id = _check_org(get_org.name) - - level1_id = _check_org(item.workflowlevel1.level1_uuid) - # set payload and deliver to tables - payload = {'level2_uuid': item.level2_uuid, 'name': item.name, 'organization': org_id, 'workflowlevel1': level1_id} - send = _send_to_tables(section="workflowlevel2", payload=payload, id=item.id) - - print send - - -def run(): - print "Running Script..." - try: - update_level1() - except Exception as e: - print '%s (%s)' % (e.message, type(e)) - - try: - update_level2() - except Exception as e: - print '%s (%s)' % (e.message, type(e)) - diff --git a/scripts/update_checklist.py b/scripts/update_checklist.py deleted file mode 100755 index 6bca0c781..000000000 --- a/scripts/update_checklist.py +++ /dev/null @@ -1,52 +0,0 @@ -""" -import json data from API -IMPORTANT!! you must turn off pagination for this to work from a URL and get all -country records -Install module django-extensions -Runs twice via function calls at bottom once -""" -from django.db import connection, transaction - -cursor = connection.cursor() -from os.path import exists -import csv -import unicodedata -import sys -import urllib2 -from datetime import date -from workflow.models import ProjectAgreement, Checklist, ChecklistItem - -def run(): - print "Uploading Country Admin data" - - -def getAllData(): - #get all the projects and loop over them - getProjects = ProjectAgreement.objects.all() - for item in getProjects: - #if the project doesn't have a checklist create one - try: - get_checklist = Checklist.objects.get(agreement=item) - except Checklist.DoesNotExist: - get_checklist = Checklist(agreement=item) - get_checklist.save() - - updateItems(get_checklist) - #if it does update the items in the checklist to include all the new globals - updateItems(get_checklist) - print item - -def updateItems(checklist): - - get_checklist = Checklist.objects.get(id=checklist.id) - get_globals = ChecklistItem.objects.all().filter(global_item=True) - for item in get_globals: - look_for_existing = ChecklistItem.objects.all().filter(checklist=get_checklist, item=item) - print item - if look_for_existing: - print "dupe do nothing" - else: - ChecklistItem.objects.create(checklist=get_checklist,item=item.item) - print item - -getAllData() \ No newline at end of file diff --git a/scripts/workflow_migration.py b/scripts/workflow_migration.py deleted file mode 100644 index dea2b5a00..000000000 --- a/scripts/workflow_migration.py +++ /dev/null @@ -1,70 +0,0 @@ -from django.apps import apps -from django.db import connection - - -app_models = apps.get_app_config('workflow').get_models() - -#rename the app tables from the old activitydb to workflow -def run(): - - print "Migration" - - for app in app_models: - name = str(app._meta.db_table) - new_appname = "tola_activity." + name - temp = name.split("_") - old_appname = "tola_activity.activitydb_" + temp[1] - - sql_query = "RENAME TABLE %s TO %s" % (old_appname,new_appname) - - print sql_query - #catch any existing tables - try: - # Renaming model from 'Foo' to 'Bar' - with connection.cursor() as cursor: - cursor.execute(sql_query) - except: - "Table Already Exists" - - name_list = [ - 'program_country', - 'program_fund_code', - 'program_sector', - 'program_user_access', - 'projectagreement_evaluate', - 'projectagreement_capacity', - 'projectagreement_stakeholder', - 'projectagreement_site', - 'projectcomplete_site', - 'projectcomplete_stakeholder', - 'quantitativeoutputs', - 'stakeholder_contact', - 'tolauser_countries' - ] - - for name in name_list: - old_appname = "tola_activity.activitydb_" + name - new_appname = "tola_activity.workflow_" + name - sql_query = "RENAME TABLE %s TO %s" % (old_appname, new_appname) - try: - # Renaming model from 'Foo' to 'Bar' - with connection.cursor() as cursor: - cursor.execute(sql_query) - except: - "Table Already Exists" - - # rename formlibrary tables - try: - # Renaming model from 'Foo' to 'Bar' - with connection.cursor() as cursor: - cursor.execute("RENAME TABLE activitydb_beneficiary TO formlibrary_beneficiary") - cursor.execute("RENAME TABLE activitydb_beneficiary_distribution TO formlibrary_beneficiary_distribution") - cursor.execute("RENAME TABLE activitydb_beneficiary_program TO fwormlibrary_beneficiary_program") - cursor.execute("RENAME TABLE activitydb_beneficiary_training TO formlibrary_beneficiary_training") - cursor.execute("RENAME TABLE activitydb_trainingattendance TO formlibrary_trainingattendance") - cursor.execute("RENAME TABLE activitydb_distribution TO formlibrary_distribution") - except: - "Table Already Exists" - - -