diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1f5afb9..6350c7e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,14 @@ Change Log ------ +3.1.0 +===== + +`PR:113 add new delete/patch script _` + +* added a new script to facilitating deleting the same field(s) for multiple items or setting the status of the items to 'deleted' +* added tests + 3.0.2 ===== diff --git a/pyproject.toml b/pyproject.toml index 45c3ee9..db7fdb8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dcicwrangling" -version = "3.0.2" +version = "3.1.0" description = "Scripts and Jupyter notebooks for 4DN wrangling" authors = ["4DN-DCIC Team "] license = "MIT" diff --git a/scripts/delete_item_or_fields.py b/scripts/delete_item_or_fields.py new file mode 100644 index 0000000..900d96e --- /dev/null +++ b/scripts/delete_item_or_fields.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +''' +Given a list of item IDs or search result will: + 1. given one of more values in the '--fields' options delete those fields from the item + 2. if no fields option then set the status of the item(s) to deleted +Useful for deleting multiple fields or changing item status to deleted +NOTE: can use patch_field_for_many_items script to delete a single field for many items +but this script is more direct/flexible in some ways +''' +import argparse +from dcicutils.ff_utils import get_metadata, delete_metadata, delete_field +from functions import script_utils as scu + + +def get_args(args): + parser = argparse.ArgumentParser( + parents=[scu.create_input_arg_parser(), scu.create_ff_arg_parser()], + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument('--fields', + nargs='+', + help="With this option these fields will be removed from the items.") + args = parser.parse_args(args) + return args + + +def main(): # pragma: no cover + args = get_args() + auth = scu.authenticate(key=args.key, keyfile=args.keyfile, env=args.env) + dry_run = True + if args.dbupdate: + dry_run = False + + print('#', auth.get('server')) + id_list = scu.get_item_ids_from_args(args.input, auth, args.search) + del_flds = None + if args.fields: + fields = args.fields + del_flds = ','.join(fields) + problems = [] + for iid in id_list: + try: + get_metadata(iid, auth, add_on='frame=object') + except Exception: + problems.append(iid) + continue + + if del_flds: + # we have field(s) that we want to delete from provided items + print(f"Will delete {del_flds} from {iid}") + if dry_run: + print("DRY RUN") + else: + try: + res = delete_field(iid, del_flds, auth) + status = res.get('status') + if status.lower() == 'success': + print(status) + else: + print(res) + problems.append(iid) + except Exception as e: + print(f"PROBLEM: {e}") + problems.append(iid) + else: + # we want to set the status of the items in id_list to deleted + print(f"Will set status of {iid} to DELETED") + if dry_run: + print("DRY RUN") + else: + try: + res = delete_metadata(iid, auth) + status = res.get('status') + if status.lower() == 'success': + print(status) + else: + print(res) + problems.append(iid) + except Exception as e: + print(f"PROBLEM: {e}") + problems.append(iid) + + if problems: + print('THERE WAS A PROBLEM DELETING METADATA FOR THE FOLLOWING:') + for p in problems: + print(p) + + +if __name__ == '__main__': + main() diff --git a/scripts/patch_field_for_many_items.py b/scripts/patch_field_for_many_items.py index a3c7e00..76e18d5 100644 --- a/scripts/patch_field_for_many_items.py +++ b/scripts/patch_field_for_many_items.py @@ -1,6 +1,6 @@ import sys import argparse -from dcicutils.ff_utils import get_authentication_with_server, patch_metadata, delete_field +from dcicutils.ff_utils import patch_metadata, delete_field from functions import script_utils as scu @@ -13,7 +13,8 @@ def get_args(args): help="The field to update.") parser.add_argument('value', help="The value(s) to update. Array fields need \"''\" surround \ - even if only a single value i.e. \"'value here'\" or \"'v1' 'v2'\"") + even if only a single value i.e. \"'value here'\" or \"'v1' 'v2'\" \ + NOTE: can delete field by passing val *delete*") parser.add_argument('--isarray', default=False, action='store_true', diff --git a/tests/test_delete_item_or_fields.py b/tests/test_delete_item_or_fields.py new file mode 100644 index 0000000..9626766 --- /dev/null +++ b/tests/test_delete_item_or_fields.py @@ -0,0 +1,149 @@ +import pytest +from scripts import delete_item_or_fields as diof + + +def test_diof_get_args_required_default(): + defaults = { + 'dbupdate': False, + 'env': None, + 'key': None, + 'keyfile': 'keypairs.json', + 'search': False, + } + args = diof.get_args(['i']) + for k, v in defaults.items(): + assert getattr(args, k) == v + assert args.input == ['i'] + + +def test_diof_get_args_missing_required(capsys): + with pytest.raises(SystemExit) as pe: + diof.get_args([]) + out = capsys.readouterr()[0] + assert 'error: the following arguments are required: input' in out + assert pe.type == SystemExit + assert pe.value.code == 2 + + +class MockedNamespace(object): + def __init__(self, dic): + for k, v in dic.items(): + setattr(self, k, v) + + +@pytest.fixture +def mocked_args_dbupd_is_false(): + return MockedNamespace( + { + 'key': None, + 'keyfile': None, + 'env': 'prod', + 'dbupdate': False, + 'search': False, + 'input': ['id1', 'id2'], + 'fields': None + } + ) + + +@pytest.fixture +def mocked_args_dbupd_is_true(): + return MockedNamespace( + { + 'key': None, + 'keyfile': None, + 'env': 'prod', + 'dbupdate': True, + 'search': False, + 'input': ['id1', 'id2'], + 'fields': None + } + ) + + +@pytest.fixture +def mocked_args_w_fields_dry_run(): + return MockedNamespace( + { + 'key': None, + 'keyfile': None, + 'env': 'prod', + 'dbupdate': False, + 'search': False, + 'input': ['id1', 'id2'], + 'fields': ['aliases'], + } + ) + + +@pytest.fixture +def mocked_args_w_fields_dbudate(): + return MockedNamespace( + { + 'key': None, + 'keyfile': None, + 'env': 'prod', + 'dbupdate': True, + 'search': False, + 'input': ['id1', 'id2'], + 'fields': ['aliases'], + } + ) +def test_diof_main_dryrun_no_fields(mocker, capsys, mocked_args_dbupd_is_false, auth): + iids = ['id1', 'id2'] + mocker.patch('scripts.delete_item_or_fields.get_args', return_value=mocked_args_dbupd_is_false) + mocker.patch('scripts.delete_item_or_fields.scu.authenticate', return_value=auth) + mocker.patch('scripts.delete_item_or_fields.scu.get_item_ids_from_args', return_value=iids) + mocker.patch('scripts.delete_item_or_fields.get_metadata', side_effect=[None, None]) + diof.main() + out = capsys.readouterr()[0] + for i in iids: + s = "Will set status of %s to DELETED\nDRY RUN" % i + assert s in out + + +def test_diof_main_dryrun_w_fields(mocker, capsys, mocked_args_w_fields_dry_run, auth): + iids = ['id1', 'id2'] + mocker.patch('scripts.delete_item_or_fields.get_args', return_value=mocked_args_w_fields_dry_run) + mocker.patch('scripts.delete_item_or_fields.scu.authenticate', return_value=auth) + mocker.patch('scripts.delete_item_or_fields.scu.get_item_ids_from_args', return_value=iids) + mocker.patch('scripts.delete_item_or_fields.get_metadata', side_effect=[None, None]) + diof.main() + out = capsys.readouterr()[0] + for i in iids: + s = "Will delete aliases from %s\nDRY RUN" % i + assert s in out + + +def test_diof_main_dbupdate_delstatus_items(mocker, capsys, mocked_args_dbupd_is_true, auth): + iids = ['id1', 'id2'] + resp1 = {'status': 'success'} + resp2 = {'status': 'error', 'description': "access denied"} + mocker.patch('scripts.delete_item_or_fields.get_args', return_value=mocked_args_dbupd_is_true) + mocker.patch('scripts.delete_item_or_fields.scu.authenticate', return_value=auth) + mocker.patch('scripts.delete_item_or_fields.scu.get_item_ids_from_args', return_value=iids) + mocker.patch('scripts.delete_item_or_fields.get_metadata', side_effect=[None, None]) + mocker.patch('scripts.delete_item_or_fields.delete_metadata', side_effect=[resp1, resp2]) + diof.main() + out = capsys.readouterr()[0] + s1 = "Will set status of %s to DELETED\nsuccess" % iids[0] + s2 = "Will set status of %s to DELETED\n{'status': 'error', 'description': 'access denied'}" %iids[1] + assert s1 in out + assert s2 in out + + +def test_diof_main_dbupdate_delfields(mocker, capsys, mocked_args_w_fields_dbudate, auth): + iids = ['id1', 'id2'] + resp1 = {'status': 'success'} + resp2 = {'status': 'error', 'description': "property not found"} + mocker.patch('scripts.delete_item_or_fields.get_args', return_value=mocked_args_w_fields_dbudate) + mocker.patch('scripts.delete_item_or_fields.scu.authenticate', return_value=auth) + mocker.patch('scripts.delete_item_or_fields.scu.get_item_ids_from_args', return_value=iids) + mocker.patch('scripts.delete_item_or_fields.get_metadata', side_effect=[None, None]) + mocker.patch('scripts.delete_item_or_fields.delete_field', side_effect=[resp1, resp2]) + diof.main() + out = capsys.readouterr()[0] + s1 = "Will delete aliases from %s\nsuccess" % iids[0] + s2 = "Will delete aliases from %s\n{'status': 'error', 'description': 'property not found'}" %iids[1] + assert s1 in out + assert s2 in out