Merge branch 'main' into des-2432

conf/env_files/designsafe.sample.env

@@ -74,6 +74,7 @@ AGAVE_CLIENT_SECRET=
 AGAVE_SUPER_TOKEN=
 
 AGAVE_STORAGE_SYSTEM=
+AGAVE_WORKING_SYSTEM=
 AGAVE_JWT_PUBKEY=
 AGAVE_JWT_ISSUER=
 AGAVE_JWT_HEADER=

designsafe/LoginTest.py

@@ -1,7 +1,7 @@
 #python manage.py test designsafe.LoginTest --settings=designsafe.settings.test_settings
 from django.test import TestCase, RequestFactory
 from django.contrib.auth import get_user_model
-from django.core.urlresolvers import reverse
+from django.urls import reverse
 import mock
 from designsafe.apps.auth.models import AgaveOAuthToken
 from designsafe.apps.auth.tasks import check_or_create_agave_home_dir

designsafe/apps/accounts/forms.py

@@ -2,7 +2,7 @@
 import logging
 from django import forms
 from django.contrib.auth import get_user_model
-from django.core.urlresolvers import reverse
+from django.urls import reverse
 from django.utils.translation import ugettext as _
 from django.utils.safestring import mark_safe
 from django.utils.html import escape

designsafe/apps/accounts/migrations/0001_initial.py

@@ -18,7 +18,7 @@ class Migration(migrations.Migration):
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                 ('ethnicity', models.CharField(max_length=255)),
                 ('gender', models.CharField(max_length=255)),
-                ('user', models.OneToOneField(related_name='profile', to=settings.AUTH_USER_MODEL)),
+                ('user', models.OneToOneField(related_name='profile', to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)),
             ],
         ),
     ]

designsafe/apps/accounts/migrations/0002_notificationpreferences.py

@@ -18,7 +18,7 @@ class Migration(migrations.Migration):
             fields=[
                 ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                 ('announcements', models.BooleanField(default=True, help_text='Receive occasional announcements from DesignSafe')),
-                ('user', models.OneToOneField(related_name='notification_preferences', to=settings.AUTH_USER_MODEL)),
+                ('user', models.OneToOneField(related_name='notification_preferences', to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)),
             ],
         ),
     ]

designsafe/apps/accounts/models.py

@@ -61,15 +61,15 @@ def __str__(self):
 
 
 class DesignSafeProfile(models.Model):
-    user = models.OneToOneField(settings.AUTH_USER_MODEL, related_name='profile')
+    user = models.OneToOneField(settings.AUTH_USER_MODEL, related_name='profile', on_delete=models.CASCADE)
     ethnicity = models.CharField(max_length=255)
     gender = models.CharField(max_length=255)
     agree_to_account_limit = models.DateTimeField(auto_now_add=True, null=True)
     bio = models.CharField(max_length=4096, default=None, null=True, blank=True)
     website = models.CharField(max_length=256, default=None, null=True, blank=True)
     orcid_id = models.CharField(max_length=256, default=None, null=True, blank=True)
     nh_interests = models.ManyToManyField(DesignSafeProfileNHInterests)
-    nh_interests_primary = models.ForeignKey(DesignSafeProfileNHInterests, related_name='nh_interests_primary', null=True)
+    nh_interests_primary = models.ForeignKey(DesignSafeProfileNHInterests, related_name='nh_interests_primary', null=True, on_delete=models.CASCADE)
     nh_technical_domains = models.ManyToManyField(DesignSafeProfileNHTechnicalDomains)
     professional_level = models.CharField(max_length=256, default=None, null=True)
     research_activities = models.ManyToManyField(DesignSafeProfileResearchActivities)
@@ -86,7 +86,7 @@ def send_mail(self, subject, body=None):
 
 class NotificationPreferences(models.Model):
     user = models.OneToOneField(settings.AUTH_USER_MODEL,
-                                related_name='notification_preferences')
+                                related_name='notification_preferences', on_delete=models.CASCADE)
     announcements = models.BooleanField(
         default=True,
         verbose_name=_('Announcements: to communicate EF Workshops, NHERI Newsletter, Student Opportunities, etc.'))

designsafe/apps/accounts/templates/designsafe/apps/accounts/register.html

@@ -26,6 +26,12 @@ <h3 class="text-center">
             to request an account.
             Once your account has been approved, you will then be able to log into DesignSafe.
           </h3>
+          <p>
+          For citizens of countries designated as “countries of concern” by United States government 
+          regulations, additional documentation in the form of a currently valid visa and passport will 
+          be requested after the account registration form has been submitted. Citizens of these countries 
+          are only eligible to have accounts while legally residing in the United States.
+        </p>
     </div>
   </div>
 

designsafe/apps/accounts/tests.py

@@ -1,7 +1,7 @@
 from django.test import TestCase
 from django.contrib.auth import get_user_model, signals
 from django.contrib.auth.models import Permission
-from django.core.urlresolvers import reverse
+from django.urls import reverse
 from unittest import skip
 from mock import patch
 import pytest

designsafe/apps/accounts/urls.py

@@ -1,5 +1,5 @@
 from django.conf.urls import include, url
-from django.core.urlresolvers import reverse
+from django.urls import reverse
 from django.utils.translation import ugettext_lazy as _
 from designsafe.apps.accounts import views
 

designsafe/apps/accounts/views.py

@@ -3,10 +3,9 @@
 from django.contrib.auth import get_user_model, logout
 from django.contrib.auth.decorators import login_required, permission_required
 from django.core.exceptions import ObjectDoesNotExist
-from django.core.urlresolvers import reverse
+from django.urls import reverse
 from django.db.models import Q
 from django.http import HttpResponse, HttpResponseRedirect
-from django.shortcuts import render_to_response
 from django.utils.translation import ugettext_lazy as _
 from designsafe.apps.accounts import forms, integrations
 from designsafe.apps.accounts.models import (NEESUser, DesignSafeProfile,
@@ -441,7 +440,9 @@ def email_confirmation(request, code=None):
                 user = tas.get_user(username=username)
                 if tas.verify_user(user['id'], code, password=password):
                     logger.info('TAS Account activation succeeded.')
-                    check_or_create_agave_home_dir.apply(args=(user["username"],))
+                    from django.conf import settings
+                    check_or_create_agave_home_dir.apply_async(args=(user.username, settings.AGAVE_STORAGE_SYSTEM))
+                    check_or_create_agave_home_dir.apply_async(args=(user.username, settings.AGAVE_WORKING_SYSTEM))
                     return HttpResponseRedirect(reverse('designsafe_accounts:manage_profile'))
                 else:
                     messages.error(request,

designsafe/apps/api/datafiles/handlers.py

@@ -7,7 +7,7 @@
 from designsafe.apps.api.datafiles.operations import shared_operations
 from designsafe.apps.api.exceptions import ApiException
 from django.core.exceptions import PermissionDenied
-from django.core.urlresolvers import reverse
+from django.urls import reverse
 
 logger = logging.getLogger(__name__)
 

designsafe/apps/api/datafiles/views.py

@@ -8,6 +8,7 @@
 from designsafe.apps.api.datafiles.notifications import notify
 from designsafe.apps.api.datafiles.models import DataFilesSurveyResult, DataFilesSurveyCounter
 from designsafe.apps.api.views import BaseApiView
+from designsafe.apps.api.agave import service_account
 from dropbox.exceptions import AuthError as DropboxAuthError
 from google.auth.exceptions import GoogleAuthError
 from requests.exceptions import HTTPError
@@ -32,6 +33,8 @@ def get_client(user, api):
 
 class DataFilesView(BaseApiView):
     def get(self, request, api, operation=None, scheme='private', system=None, path=''):
+
+        doi = request.GET.get('doi', None)
 
         metrics.info('Data Depot',
                      extra={
@@ -44,6 +47,7 @@ def get(self, request, api, operation=None, scheme='private', system=None, path=
                              'api': api,
                              'systemId': system,
                              'filePath': path,
+                             'doi': doi,
                              'query': request.GET.dict()}
                      })
 
@@ -53,7 +57,7 @@ def get(self, request, api, operation=None, scheme='private', system=None, path=
             except AttributeError:
                 raise resource_unconnected_handler(api)
         elif api == 'agave':
-            client = get_user_model().objects.get(username='envision').agave_oauth.client
+            client = service_account()
         else:
             return JsonResponse({'message': 'Please log in to access this feature.'}, status=403)
 
@@ -69,6 +73,7 @@ def get(self, request, api, operation=None, scheme='private', system=None, path=
     def put(self, request, api, operation=None, scheme='private', system=None, path='/'):
 
         body = json.loads(request.body)
+        doi = request.GET.get('doi', None)
 
         metrics.info('Data Depot',
                      extra={
@@ -82,7 +87,8 @@ def put(self, request, api, operation=None, scheme='private', system=None, path=
                              'scheme': scheme,
                              'system': system,
                              'path': path,
-                             'body': body
+                             'body': body,
+                             'doi': doi
                          }
                      })
 

designsafe/apps/api/decorators.py

@@ -4,7 +4,6 @@
 import logging
 from functools import wraps
 from base64 import b64decode
-from django.utils.six import text_type
 from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.contrib.auth import login
@@ -52,7 +51,7 @@ def _get_jwt_payload(request):
     :rtype: str
     """
     payload = request.META.get(settings.AGAVE_JWT_HEADER)
-    if payload and isinstance(payload, text_type):
+    if payload and isinstance(payload, str):
         # Header encoding (see RFC5987)
         payload = payload.encode('iso-8859-1')
 
@@ -104,8 +103,7 @@ def decorated_function(request, *args, **kwargs):
             user = None
 
         if user is not None:
-            user.backend = 'django.contrib.auth.backends.ModelBackend',
-            login(request, user)
+            login(request, user, backend="django.contrib.auth.backends.ModelBackend")
 
         return func(request, *args, **kwargs)
 

designsafe/apps/api/notifications/tests.py

@@ -10,7 +10,7 @@
 from urllib.parse import urlencode
 from unittest import skip
 from django.dispatch import receiver
-from django.core.urlresolvers import reverse
+from django.urls import reverse
 from designsafe.apps.api.notifications.models import Notification
 from .receivers import send_notification_ws
 

designsafe/apps/api/notifications/views/api.py

@@ -1,7 +1,7 @@
 import logging
 from django.http.response import HttpResponseBadRequest
 from django.http import HttpResponse
-from django.core.urlresolvers import reverse
+from django.urls import reverse
 from django.shortcuts import render
 
 from designsafe.apps.api.notifications.models import Notification

designsafe/apps/api/notifications/views/webhooks.py

@@ -1,7 +1,7 @@
 from django.http.response import HttpResponseBadRequest
 from django.core.exceptions import ObjectDoesNotExist
 from django.contrib.auth import get_user_model
-from django.core.urlresolvers import reverse
+from django.urls import reverse
 from django.views.decorators.csrf import csrf_exempt
 from django.utils.decorators import method_decorator
 from django.shortcuts import render

designsafe/apps/api/projects/tests.py

@@ -1,12 +1,15 @@
 from designsafe.apps.api.projects.fixtures import exp_instance_meta, exp_instance_resp, exp_entity_meta, exp_entity_json
+import pytest
 
+@pytest.mark.django_db
 def test_project_instance_get(client, mock_agave_client, authenticated_user):
     mock_agave_client.meta.getMetadata.return_value = exp_instance_meta
     resp = client.get('/api/projects/1052668239654088215-242ac119-0001-012/')
     actual = resp.json()
     expected = exp_instance_resp
     assert actual == expected
 
+@pytest.mark.django_db
 def test_project_meta_all(client, mock_agave_client, authenticated_user):
     mock_agave_client.meta.getMetadata.return_value = exp_instance_meta
     mock_agave_client.meta.listMetadata.return_value = exp_entity_meta

designsafe/apps/api/projects/views.py

@@ -3,7 +3,7 @@
 import logging
 import json
 from celery import group, chain
-from django.core.urlresolvers import reverse
+from django.urls import reverse
 from django.conf import settings
 from django.http.response import HttpResponseForbidden
 from django.http import JsonResponse, HttpResponseBadRequest

designsafe/apps/api/publications/search_utils.py

@@ -177,4 +177,4 @@ def search_string_query(search_string):
                                                           'project.value.projectType',
                                                           'project.value.dataType'])
     q2 = Q({'term': {'projectId._exact': search_string}})
-    return q1 | q2 | author_query(search_string)
+    return q1 | q2 | author_query(f"\"{search_string}\"")

designsafe/apps/api/search/tests.py

@@ -2,7 +2,7 @@
 from django.test import TestCase
 from django.contrib.auth import get_user_model
 from django.conf import settings
-from django.core.urlresolvers import reverse
+from django.urls import reverse
 from designsafe.apps.api.search.views import SearchView
 import json
 

designsafe/apps/api/tasks.py

@@ -7,7 +7,7 @@
 import urllib.request, urllib.parse, urllib.error
 from datetime import datetime
 from celery import shared_task
-from django.core.urlresolvers import reverse
+from django.urls import reverse
 from django.contrib.auth import get_user_model
 from pytas.models import User as TASUser
 from django.conf import settings
@@ -670,6 +670,8 @@ def amend_publication_data(self, project_id, amendments=None, authors=None, revi
     from designsafe.apps.projects.managers import publication as PublicationManager
     from designsafe.libs.fedora.fedora_operations import amend_project_fedora, ingest_project_experimental
     from designsafe.libs.fedora.sim_operations import ingest_project_sim
+    from designsafe.libs.fedora.fr_operations import ingest_project_fr
+    from designsafe.libs.fedora.hyb_sim_operations import ingest_project_hyb_sim
     try:
         amended_pub = PublicationManager.amend_publication(project_id, amendments, authors, revision)
         PublicationManager.amend_datacite_doi(amended_pub)
@@ -680,6 +682,10 @@ def amend_publication_data(self, project_id, amendments=None, authors=None, revi
             ingest_project_experimental(project_id, version=revision, amend=True)
         if amended_pub.project.value.projectType == 'simulation':
             ingest_project_sim(project_id, version=revision, amend=True)
+        if amended_pub.project.value.projectType == 'hybrid_simulation':
+            ingest_project_hyb_sim(project_id, version=revision, amend=True)
+        if amended_pub.project.value.projectType == 'field_recon':
+            ingest_project_fr(project_id, version=revision, amend=True)
     except Exception as exc:
         logger.error('Proj Id: %s. %s', project_id, exc, exc_info=True)
         raise self.retry(exc=exc)
@@ -780,41 +786,14 @@ def save_to_fedora(self, project_id, revision=None):
             from designsafe.libs.fedora.sim_operations import ingest_project_sim
             ingest_project_sim(project_id, version=revision)
             return
-
-        _root = os.path.join('/corral-repl/tacc/NHERI/published', project_id)
-        fedora_base = 'http://fedoraweb01.tacc.utexas.edu:8080/fcrepo/rest/publications_01'
-        res = requests.get(fedora_base)
-        if res.status_code == 404 or res.status_code == 410:
-            requests.put(fedora_base)
-
-        fedora_project_base = ''.join([fedora_base, '/', project_id])
-        res = requests.get(fedora_project_base)
-        if res.status_code == 404 or res.status_code == 410:
-            requests.put(fedora_project_base)
-
-        headers = {'Content-Type': 'text/plain'}
-        #logger.debug('walking: %s', _root)
-        for root, dirs, files in os.walk(_root):
-            for name in files:
-                mime = magic.Magic(mime=True)
-                headers['Content-Type'] = mime.from_file(os.path.join(root, name))
-                #files
-                full_path = os.path.join(root, name)
-                _path = full_path.replace(_root, '', 1)
-                _path = _path.replace('[', '-')
-                _path = _path.replace(']', '-')
-                url = ''.join([fedora_project_base, urllib.parse.quote(_path)])
-                #logger.debug('uploading: %s', url)
-                with open(os.path.join(root, name), 'rb') as _file:
-                    requests.put(url, data=_file, headers=headers)
-
-            for name in dirs:
-                #dirs
-                full_path = os.path.join(root, name)
-                _path = full_path.replace(_root, '', 1)
-                url = ''.join([fedora_project_base, _path])
-                #logger.debug('creating: %s', _path)
-                requests.put(url)
+        if pub.project.value.projectType == 'hybrid_simulation':
+            from designsafe.libs.fedora.hyb_sim_operations import ingest_project_hyb_sim
+            ingest_project_hyb_sim(project_id, version=revision)
+            return
+        if pub.project.value.projectType == 'field_recon':
+            from designsafe.libs.fedora.fr_operations import ingest_project_fr
+            ingest_project_fr(project_id, version=revision)
+            return
 
     except Exception as exc:
         logger.error('Proj Id: %s. %s', project_id, exc)

designsafe/apps/api/urls.py

@@ -4,7 +4,7 @@
 from django.http import JsonResponse
 
 urlpatterns = [
-    url(r'^projects/', include('designsafe.apps.api.projects.urls',
+    url(r'^projects/', include(('designsafe.apps.api.projects.urls', 'designsafe.apps.api.projects'),
                                namespace='ds_projects_api')),
 
     url(r'^datafiles/', include('designsafe.apps.api.datafiles.urls')),
@@ -13,6 +13,6 @@
     url(r'^logger/$', LoggerApi.as_view(), name='logger'),
     url(r'^notifications/', include('designsafe.apps.api.notifications.urls')),
     url(r'^users/', include('designsafe.apps.api.users.urls')),
-    url(r'^search/', include('designsafe.apps.api.search.urls', namespace="ds_search_api")),
+    url(r'^search/', include(('designsafe.apps.api.search.urls', 'designsafe.apps.api.search'), namespace="ds_search_api")),
     url(r'^licenses/', include('designsafe.apps.api.licenses.urls'))
 ]

designsafe/apps/applications/urls.py

@@ -1,5 +1,5 @@
 from django.conf.urls import include, url
-from django.core.urlresolvers import reverse
+from django.urls import reverse
 from django.utils.translation import ugettext_lazy as _
 from designsafe.apps.applications import views
 

designsafe/apps/applications/views.py

@@ -47,7 +47,7 @@ def call_api(request, service):
         :param request: http request from angular service
         :returns: JSON response for agave call
     """
-    if request.user.is_authenticated():
+    if request.user.is_authenticated:
         try:
             token = request.user.agave_oauth
             agave = Agave(api_server=settings.AGAVE_TENANT_BASEURL,

designsafe/apps/auth/backends.py

@@ -50,7 +50,7 @@ def __init__(self):
 
     # Create an authentication method
     # This is called by the standard Django login procedure
-    def authenticate(self, username=None, password=None, request=None, **kwargs):
+    def authenticate(self, request, username=None, password=None, **kwargs):
         user = None
         if username is not None and password is not None:
             tas_user = None