Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

task/DES-2614: Track fileObjs when categorizing files #1129

Merged
merged 9 commits into from
Dec 13, 2023
Merged

task/DES-2614: Track fileObjs when categorizing files #1129

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
02a4fa0
Track fileObjs when categorizing files
Nov 16, 2023
2b45a4e
update test fixtures
Nov 16, 2023
60ec342
add UUIDs and remove orphan file tags
Dec 11, 2023
ea169f1
Merge branch 'main' into task/DES-2614--track-fileobjs-in-projects
jarosenb Dec 11, 2023
bb73aab
fix indexing bug
Dec 12, 2023
169ffdc
prevent indexing projects without UUIDs
Dec 12, 2023
bd8d673
Merge branch 'main' into task/DES-2614--track-fileobjs-in-projects
jarosenb Dec 12, 2023
7a69e31
Disable file tag cleanup-- could result in unwanted results if file_o…
Dec 12, 2023
d424635
Merge branch 'main' into task/DES-2614--track-fileobjs-in-projects
jarosenb Dec 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions designsafe/apps/api/projects/fixtures.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@
'experiments': ['7091290346386681365-242ac118-0001-012',
'3849330284117683735-242ac119-0001-012'],
'fileTags': [],
'fileObjs': [],
'files': ['1755305162898796054-242ac112-0001-002',
'1874008375186297321-242ac112-0001-002'],
'project': ['1052668239654088215-242ac119-0001-012'],
Expand Down Expand Up @@ -196,6 +197,7 @@
{'fileUuid': '1871173696770937321-242ac112-0001-002',
'path': '/00004.jpg',
'tagName': 'Video'}],
'fileObjs': [],
'files': ['4352778795703407081-242ac114-0001-002',
'4360552686509167081-242ac114-0001-002',
'4352735846030447081-242ac114-0001-002',
Expand All @@ -222,6 +224,7 @@
'value': {'description': 'Hello WOrld',
'experiments': [],
'fileTags': [],
'fileObjs': [],
'files': [],
'project': ['1052668239654088215-242ac119-0001-012'],
'title': 'A new Unassociated Report'}},
Expand Down Expand Up @@ -253,6 +256,7 @@
'description': '',
'experiments': ['7091290346386681365-242ac118-0001-012'],
'fileTags': [],
'fileObjs': [],
'files': ['7120165510589246999-242ac113-0001-002'],
'project': ['1052668239654088215-242ac119-0001-012'],
'refs': [None],
Expand Down Expand Up @@ -363,6 +367,7 @@
'value': {'description': 'Truck',
'experiments': [],
'fileTags': [],
'fileObjs': [],
'files': [],
'modelConfigs': [],
'project': ['1052668239654088215-242ac119-0001-012'],
Expand Down Expand Up @@ -412,6 +417,7 @@
'experiments': ['3849330284117683735-242ac119-0001-012',
'7091290346386681365-242ac118-0001-012'],
'fileTags': [],
'fileObjs': [],
'files': ['4354582681967727081-242ac114-0001-002',
'4358448152534127081-242ac114-0001-002'],
'modelConfigs': ['3339522405406469655-242ac11a-0001-012'],
Expand Down Expand Up @@ -456,6 +462,7 @@
'experiments': ['3849330284117683735-242ac119-0001-012',
'7091290346386681365-242ac118-0001-012'],
'fileTags': [],
'fileObjs': [],
'files': ['2993276852523897321-242ac112-0001-002',
'7112348674405494295-242ac113-0001-002'],
'modelConfigs': ['3339522405406469655-242ac11a-0001-012'],
Expand Down Expand Up @@ -487,6 +494,7 @@
'value': {'description': 'Test',
'experiments': ['7091290346386681365-242ac118-0001-012'],
'fileTags': [],
'fileObjs': [],
'files': ['7119435370443894295-242ac113-0001-002'],
'project': ['1052668239654088215-242ac119-0001-012'],
'title': 'Project Report'}},
Expand All @@ -509,6 +517,7 @@
'eventType': 'other',
'experiments': [],
'fileTags': [],
'fileObjs': [],
'files': [],
'modelConfigs': [],
'project': ['1052668239654088215-242ac119-0001-012'],
Expand All @@ -533,6 +542,7 @@
'eventType': 'other',
'experiments': [],
'fileTags': [],
'fileObjs': [],
'files': [],
'modelConfigs': [],
'project': ['1052668239654088215-242ac119-0001-012'],
Expand Down Expand Up @@ -604,6 +614,7 @@
'value': {'description': 'This report needs a description...',
'experiments': [],
'fileTags': [],
'fileObjs': [],
'files': ['2621891030438777321-242ac112-0001-002'],
'project': ['1052668239654088215-242ac119-0001-012'],
'title': 'Findings From Series B'}},
Expand Down Expand Up @@ -632,6 +643,7 @@
'value': {'description': 'This is where the description goes. It is useful for a number of reasons, but you should always remember to speyul check your work.',
'experiments': ['3849330284117683735-242ac119-0001-012'],
'fileTags': [],
'fileObjs': [],
'files': ['7120337313576054295-242ac113-0001-002'],
'project': ['1052668239654088215-242ac119-0001-012'],
'title': 'Findings From Series A'}},
Expand Down Expand Up @@ -664,6 +676,7 @@
'experiments': ['3849330284117683735-242ac119-0001-012'],
'fileTags': [{'fileUuid': '1873922475840377321-242ac112-0001-002',
'tagName': 'Visualization'}],
'fileObjs': [],
'files': ['7120165514884214295-242ac113-0001-002'],
'project': ['1052668239654088215-242ac119-0001-012'],
'refs': [{'reference': 'Data References 1',
Expand Down Expand Up @@ -723,6 +736,7 @@
{'fileUuid': '1873922475840377321-242ac112-0001-002',
'path': '/00001.jpg',
'tagName': 'Model Drawing'}],
'fileObjs': [],
'files': ['7137021306710659561-242ac114-0001-002',
'303745108476629482-242ac112-0001-002',
'6923074490394022377-242ac114-0001-002'],
Expand Down Expand Up @@ -761,6 +775,7 @@
'value': {'description': 'We surrounded our test structure with four sensors to take pressure readings. TEST',
'experiments': ['3849330284117683735-242ac119-0001-012'],
'fileTags': [],
'fileObjs': [],
'files': ['1874523771261817321-242ac112-0001-002',
'2080682201469817321-242ac112-0001-002'],
'modelConfigs': ['7206731579572621801-242ac119-0001-012'],
Expand Down Expand Up @@ -805,6 +820,7 @@
'experiments': ['7091290346386681365-242ac118-0001-012',
'3849330284117683735-242ac119-0001-012'],
'fileTags': [],
'fileObjs': [],
'files': ['1755305162898796054-242ac112-0001-002',
'1874008375186297321-242ac112-0001-002'],
'project': ['1052668239654088215-242ac119-0001-012'],
Expand Down Expand Up @@ -868,6 +884,7 @@
'description': 'The results from our pressure sensors showed weaknesses in the tested structure.',
'eventType': 'other',
'experiments': ['3849330284117683735-242ac119-0001-012'],
'fileObjs': [],
'fileTags': [{'fileUuid': '2621891030438777321-242ac112-0001-002',
'tagName': 'Aerodynamic roughness'},
{'fileUuid': '2621891030438777321-242ac112-0001-002', 'tagName': 'Image'},
Expand Down Expand Up @@ -910,6 +927,7 @@
'value': {'description': 'Hello WOrld',
'experiments': [],
'fileTags': [],
'fileObjs': [],
'files': [],
'project': ['1052668239654088215-242ac119-0001-012'],
'title': 'A new Unassociated Report'}},
Expand Down Expand Up @@ -945,6 +963,7 @@
'description': '',
'experiments': ['7091290346386681365-242ac118-0001-012'],
'fileTags': [],
'fileObjs': [],
'files': ['7120165510589246999-242ac113-0001-002'],
'project': ['1052668239654088215-242ac119-0001-012'],
'refs': [None],
Expand Down Expand Up @@ -1061,6 +1080,7 @@
'value': {'description': 'Truck',
'experiments': [],
'fileTags': [],
'fileObjs': [],
'files': [],
'modelConfigs': [],
'project': ['1052668239654088215-242ac119-0001-012'],
Expand Down Expand Up @@ -1117,6 +1137,7 @@
'experiments': ['3849330284117683735-242ac119-0001-012',
'7091290346386681365-242ac118-0001-012'],
'fileTags': [],
'fileObjs': [],
'files': ['4354582681967727081-242ac114-0001-002',
'4358448152534127081-242ac114-0001-002'],
'modelConfigs': ['3339522405406469655-242ac11a-0001-012'],
Expand Down Expand Up @@ -1163,6 +1184,7 @@
'experiments': ['3849330284117683735-242ac119-0001-012',
'7091290346386681365-242ac118-0001-012'],
'fileTags': [],
'fileObjs': [],
'files': ['2993276852523897321-242ac112-0001-002',
'7112348674405494295-242ac113-0001-002'],
'modelConfigs': ['3339522405406469655-242ac11a-0001-012'],
Expand Down Expand Up @@ -1196,6 +1218,7 @@
'value': {'description': 'Test',
'experiments': ['7091290346386681365-242ac118-0001-012'],
'fileTags': [],
'fileObjs': [],
'files': ['7119435370443894295-242ac113-0001-002'],
'project': ['1052668239654088215-242ac119-0001-012'],
'title': 'Project Report'}},
Expand Down Expand Up @@ -1225,6 +1248,7 @@
'eventType': 'other',
'experiments': [],
'fileTags': [],
'fileObjs': [],
'files': [],
'modelConfigs': [],
'project': ['1052668239654088215-242ac119-0001-012'],
Expand Down Expand Up @@ -1256,6 +1280,7 @@
'eventType': 'other',
'experiments': [],
'fileTags': [],
'fileObjs': [],
'files': [],
'modelConfigs': [],
'project': ['1052668239654088215-242ac119-0001-012'],
Expand Down Expand Up @@ -1331,6 +1356,7 @@
'value': {'description': 'This report needs a description...',
'experiments': [],
'fileTags': [],
'fileObjs': [],
'files': ['2621891030438777321-242ac112-0001-002'],
'project': ['1052668239654088215-242ac119-0001-012'],
'title': 'Findings From Series B'}},
Expand Down Expand Up @@ -1363,6 +1389,7 @@
'value': {'description': 'This is where the description goes. It is useful for a number of reasons, but you should always remember to speyul check your work.',
'experiments': ['3849330284117683735-242ac119-0001-012'],
'fileTags': [],
'fileObjs': [],
'files': ['7120337313576054295-242ac113-0001-002'],
'project': ['1052668239654088215-242ac119-0001-012'],
'title': 'Findings From Series A'}},
Expand Down Expand Up @@ -1397,6 +1424,7 @@
'experiments': ['3849330284117683735-242ac119-0001-012'],
'fileTags': [{'fileUuid': '1873922475840377321-242ac112-0001-002',
'tagName': 'Visualization'}],
'fileObjs': [],
'files': ['7120165514884214295-242ac113-0001-002'],
'project': ['1052668239654088215-242ac119-0001-012'],
'refs': [{'reference': 'Data References 1',
Expand Down Expand Up @@ -1442,6 +1470,7 @@
'uuid': '7206731579572621801-242ac119-0001-012',
'value': {'description': 'We are testing the stability of a structure reinforced with concrete pillars. Several waves of different sizes were tested against the structure. This is the first test for using a Large Wave Flume.',
'experiments': ['3849330284117683735-242ac119-0001-012'],
'fileObjs': [],
'fileTags': [{'fileUuid': '4352735846030447081-242ac114-0001-002',
'path': '/Setup Documentation/00005.jpg',
'tagName': 'Image'},
Expand Down Expand Up @@ -1502,6 +1531,7 @@
'value': {'description': 'We surrounded our test structure with four sensors to take pressure readings. TEST',
'experiments': ['3849330284117683735-242ac119-0001-012'],
'fileTags': [],
'fileObjs': [],
'files': ['1874523771261817321-242ac112-0001-002',
'2080682201469817321-242ac112-0001-002'],
'modelConfigs': ['7206731579572621801-242ac119-0001-012'],
Expand Down
2 changes: 2 additions & 0 deletions designsafe/apps/api/tasks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -499,6 +499,8 @@ def index_projects_listing(projects):
project_dict['value']['awardNumber'] = award_number

project_dict['value'].pop('nhEventStart', None)
project_dict['value'].pop('fileObjs', None)
project_dict['value'].pop('project', None)
project_dict['value'].pop('nhEventEnd', None)
project_dict['value'].pop('referencedData', None)
project_dict['value'].pop('relatedFiles', None)
Expand Down
3 changes: 2 additions & 1 deletion designsafe/apps/data/models/agave/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -463,7 +463,8 @@ def save(self, agave_client):
else:
logger.debug('Updating Metadata: %s, with: %s', self.uuid, body)
ret = agave_client.meta.updateMetadata(uuid=self.uuid, body=body)
tasks.index_or_update_project.apply_async(args=[self.uuid], queue='api')
if self.name == "designsafe.project":
tasks.index_or_update_project.apply_async(args=[self.uuid], queue='api')

self.update(**ret)
return ret
Expand Down
22 changes: 22 additions & 0 deletions designsafe/apps/projects/models/agave/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,16 @@
from designsafe.libs.elasticsearch.exceptions import DocumentNotFound
logger = logging.getLogger(__name__)

class FileObjModel(MetadataModel):
_is_nested = True
path = fields.CharField('File path', max_length=1024, default='')
name = fields.CharField('File name', max_length=1024, default='')
system = fields.CharField('System', max_length=512, default='')
type = fields.CharField('File type', max_length=512, default='')
last_modified = fields.CharField('Last Modified', max_length=512, default='')
uuid = fields.CharField('UUID', max_length=512, default='')
length = fields.IntField('File Size', default=0)


class RelatedEntity(MetadataModel):
"""Model for entities related to projects."""
Expand All @@ -25,6 +35,18 @@ def to_body_dict(self):
for attrname, field in six.iteritems(self._meta._related_fields):
body_dict['_relatedFields'].append(attrname)
return body_dict

# TODO: uncomment when file_objs are synced.
#def save(self, ac):
# """Remove file tags if the file has been un-associated from the entity."""
# file_objs = getattr(self, 'file_objs', [])
# file_tags = getattr(self, 'file_tags', [])
#
# if file_objs and file_tags:
# filtered_tags = filter(lambda tag: bool([file for file in file_objs if tag["path"].startswith(file["path"])]), file_tags)
# self.file_tags = list(filtered_tags)
#
# return super().save(ac)

def to_datacite_json(self):
"""Serialize object to datacite JSON.
Expand Down
8 changes: 7 additions & 1 deletion designsafe/apps/projects/models/agave/experimental.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import six
from designsafe.apps.data.models.agave.base import Model as MetadataModel
from designsafe.apps.data.models.agave import fields
from designsafe.apps.projects.models.agave.base import RelatedEntity, Project
from designsafe.apps.projects.models.agave.base import RelatedEntity, Project, FileObjModel

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -44,6 +44,7 @@ class DataTag(MetadataModel):
tag_name = fields.CharField('Tag Name', max_length=512, default='')
value = fields.CharField('Value', max_length=512, default='')


class Experiment(RelatedEntity):
model_name = 'designsafe.project.experiment'
experiment_type = fields.CharField('Experiment Type', max_length=255, default='other')
Expand Down Expand Up @@ -132,6 +133,7 @@ class Analysis(RelatedEntity):
experiments = fields.RelatedObjectField(Experiment)
files = fields.RelatedObjectField(FileModel, multiple=True)
file_tags = fields.ListField('File Tags', list_cls=DataTag)
file_objs = fields.ListField('File Objects', list_cls=FileObjModel)

class ModelConfig(RelatedEntity):
model_name = 'designsafe.project.model_config'
Expand All @@ -141,6 +143,7 @@ class ModelConfig(RelatedEntity):
experiments = fields.RelatedObjectField(Experiment)
files = fields.RelatedObjectField(FileModel, multiple=True)
file_tags = fields.ListField('File Tags', list_cls=DataTag)
file_objs = fields.ListField('File Objects', list_cls=FileObjModel)

class SensorList(RelatedEntity):
model_name = 'designsafe.project.sensor_list'
Expand All @@ -152,6 +155,7 @@ class SensorList(RelatedEntity):
model_configs = fields.RelatedObjectField(ModelConfig)
files = fields.RelatedObjectField(FileModel, multiple=True)
file_tags = fields.ListField('File Tags', list_cls=DataTag)
file_objs = fields.ListField('File Objects', list_cls=FileObjModel)

class Event(RelatedEntity):
model_name = 'designsafe.project.event'
Expand All @@ -165,6 +169,7 @@ class Event(RelatedEntity):
sensor_lists = fields.RelatedObjectField(SensorList)
files = fields.RelatedObjectField(FileModel, multiple=True)
file_tags = fields.ListField('File Tags', list_cls=DataTag)
file_objs = fields.ListField('File Objects', list_cls=FileObjModel)

class Report(RelatedEntity):
model_name = 'designsafe.project.report'
Expand All @@ -174,3 +179,4 @@ class Report(RelatedEntity):
experiments = fields.RelatedObjectField(Experiment)
files = fields.RelatedObjectField(FileModel, multiple=True)
file_tags = fields.ListField('File Tags', list_cls=DataTag)
file_objs = fields.ListField('File Objects', list_cls=FileObjModel)
Loading