diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..23a4847d9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,33 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: bug +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Desktop (Optional):** + - OS: [e.g. macOS, Windows] + - Browser [e.g. chrome, safari] + - Version [e.g. 22] + + +**Additional context (Optional)** +Add any other context about the problem here. \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000..0257906b0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for a feature or enhancement +title: '' +labels: feature +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered (Optional)** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context (Optional)** +Add any other context or screenshots about the feature request here. \ No newline at end of file diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..0839f0322 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,28 @@ +## Description + +**** Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change. **** + +Fixes #(issue number) + +## Type of change + + **** Please delete options that are not relevant. **** + +- Bug fix (non-breaking change which fixes an issue) +- New feature (non-breaking change which adds functionality) +- Breaking change (fix or feature that would cause existing functionality to not work as expected) + +## How Has This Been Tested? + +**** Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. Please also list any relevant details for your test configuration. **** + + +## PR Self Evaluation +Strikethrough things that don’t make sense for your PR. + +- [ ] My code follows the agreed upon best practices +- [ ] I have commented my code, particularly in hard-to-understand areas +- [ ] I have made corresponding changes to the documentation (if needed) +- [ ] My changes generate no new warnings +- [ ] Any dependent changes have been merged and published in the appropriate modules +- [ ] I have performed a self-review of my own code diff --git a/.gitignore b/.gitignore index 32d292253..edd0de3f9 100644 --- a/.gitignore +++ b/.gitignore @@ -39,4 +39,4 @@ google-drive-key.json overrides.yml New Project Request MOUs Secure Directory Request MOUs -Service Units Purchase Request MOUs \ No newline at end of file +Service Units Purchase Request MOUs diff --git a/bootstrap/ansible/main.copyme b/bootstrap/ansible/main.copyme index df55e5305..15db4bebd 100644 --- a/bootstrap/ansible/main.copyme +++ b/bootstrap/ansible/main.copyme @@ -24,6 +24,10 @@ enable_django_debug_toolbar: false # TODO: Generate one using: `openssl rand -base64 64`. django_secret_key: +#------------------------------------------------------------------------------ +# PostgreSQL database settings +#------------------------------------------------------------------------------ + # The name of the PostgreSQL database. # TODO: For LRC, set this to 'cf_lrc_db'. db_name: cf_brc_db @@ -34,35 +38,53 @@ db_host: localhost db_admin_user: admin db_admin_passwd: '' +#------------------------------------------------------------------------------ +# Redis settings +#------------------------------------------------------------------------------ + # The password for Redis. # TODO: Replace the password. redis_passwd: '' + redis_host: localhost -# Log paths. +#------------------------------------------------------------------------------ +# Logging settings +#------------------------------------------------------------------------------ + # TODO: For LRC, use the substring 'cf_mylrc'. log_path: /var/log/user_portals/cf_mybrc portal_log_file: cf_mybrc_portal.log api_log_file: cf_mybrc_api.log + logrotate_entry_name: cf_mybrc + # TODO: Logs are only backed up if a path to an existent directory is given. log_backup_dir_path: + # TODO: For Docker environments, set this to true. stream_logs_to_stdout: false -# Apache settings. +#------------------------------------------------------------------------------ +# Apache settings +#------------------------------------------------------------------------------ + # The name of the copy of the generated WSGI template in the Apache directory. # TODO: For LRC, use the substring 'cf_mylrc'. wsgi_conf_file_name: cf_mybrc_wsgi.conf # TODO: For LRC, use the substring 'cf_lrc'. wsgi_conf_log_prefix: cf_brc -# LRC Cloudflare settings. +#------------------------------------------------------------------------------ +# LRC Cloudflare settings +#------------------------------------------------------------------------------ + # Whether the web server is behind Cloudflare. # TODO: For the LRC production deployment, enable Cloudflare, since LBL # TODO: requires that web servers visible to the Internet be placed behind it. # TODO: https://commons.lbl.gov/display/cpp/Open+Web+Server+Requirements cloudflare_enabled: false + # A list of Cloudflare's IP ranges. # TODO: Keep it up-to-date with: https://www.cloudflare.com/ips/. cloudflare_ip_ranges: [ @@ -82,41 +104,63 @@ cloudflare_ip_ranges: [ 197.234.240.0/22, 198.41.128.0/17 ] + # The name of the server, which should differ from the name of the website. # Source: See Open Web Server Requirements link above. # TODO: Set this to e.g., mylrc-local.lbl.gov for mylrc.lbl.gov if Cloudflare # TODO: is enabled. cloudflare_local_server_name: -# CILogon client settings. +#------------------------------------------------------------------------------ +# CILogon client settings +#------------------------------------------------------------------------------ + # TODO: Set these, needed only if SSO should be enabled. cilogon_app_client_id: "" cilogon_app_secret: "" -# Django Flags settings. +#------------------------------------------------------------------------------ +# Django Flags settings +#------------------------------------------------------------------------------ + +# # Note: Use uppercase True/False so that Python interprets these as booleans. + # TODO: For LRC, disable link login. flag_basic_auth_enabled: False flag_sso_enabled: True flag_link_login_enabled: True + # TODO: For LRC, disable BRC and enable LRC. flag_brc_enabled: True flag_lrc_enabled: False + # The number of the month in which users should be able to request renewal for # the next allowance year. # TODO: For LRC, set the month number to 9 (September). flag_next_period_renewal_requestable_month: 5 + +# Whether to enable UI support for users having multiple email addresses. flag_multiple_email_addresses_allowed: False + # Whether to install and enable the MOU generation package. # TODO: For BRC, enable MOU generation (requires access to the package). flag_mou_generation_enabled: False -# Portal settings. +# Whether to include a survey as part of the allowance renewal request process. +flag_renewal_survey_enabled: True + +#------------------------------------------------------------------------------ +# User-facing strings +#------------------------------------------------------------------------------ + # TODO: For LRC, use "MyLRC", "Laboratory Research Computing", "LRC", and # TODO: "Lawrencium". portal_name: "MyBRC" program_name_long: "Berkeley Research Computing" program_name_short: "BRC" + primary_cluster_name: "Savio" + # TODO: For MyLRC, use "https://it.lbl.gov/service/scienceit/high-performance-computing/lrc/". center_user_guide: "https://docs-research-it.berkeley.edu/services/high-performance-computing/user-guide/" # TODO: For MyLRC, use "https://it.lbl.gov/resource/hpc/for-users/getting-started/". @@ -124,9 +168,13 @@ center_login_guide: "https://docs-research-it.berkeley.edu/services/high-perform # TODO: For MyLRC, use "hpcshelp@lbl.gov". center_help_email: "brc-hpc-help@berkeley.edu" -# # Storage settings. -# # For more information, refer to the README. +#------------------------------------------------------------------------------ +# BRC File storage settings +#------------------------------------------------------------------------------ + +# # Backend options: 'file_system', 'google_drive' # file_storage_backend: 'file_system' + # new_project_request_mou_path: 'New Project Request MOUs/' # secure_directory_request_mou_path: 'Secure Directory Request MOUs/' # service_units_purchase_request_mou_path: 'Service Units Purchase Request MOUs/' @@ -141,40 +189,76 @@ center_help_email: "brc-hpc-help@berkeley.edu" # google_drive_private_key_file_path: '/path/to/google-drive-private-key.json' # google_drive_storage_media_root: '/' -# # MOU generator settings. -# # For more information, refer to the README. +#------------------------------------------------------------------------------ +# BRC MOU generation settings +#------------------------------------------------------------------------------ + # # TODO: For BRC deployments with access to the package for generating MOUs # # TODO: (e.g., production and staging), set the absolute path to the deploy -# # TODO: key. +# # TODO: key for the package. # mou_generator_deploy_key_path: '/path/to/id_mou_generator' -# LRC billing validation package settings (for Pip installation). +#------------------------------------------------------------------------------ +# LRC Billing validation settings +#------------------------------------------------------------------------------ + # TODO: For LRC deployments with access to the package for billing validation # TODO: (e.g., production and staging), set these. install_billing_validation_package: false # Example: "gitlab.com/user/repo_name.git" billing_validation_repo_host: "" # Create or request a deploy token. +# Reference: https://docs.gitlab.com/ee/user/project/deploy_tokens/ billing_validation_repo_username: "" billing_validation_repo_token: "" -# LRC Oracle billing database settings. -# TODO: For LRC deployments with access to Oracle, set these. +# TODO: For LRC deployments with access to the Oracle billing database, set +# TODO: these. oracle_billing_db_dsn: "" oracle_billing_db_user: "" oracle_billing_db_passwd: "" -# API settings. +#------------------------------------------------------------------------------ +# Allowance renewal survey settings +#------------------------------------------------------------------------------ + +# # Backend options: 'google_forms', 'permissive' +# renewal_survey_backend: 'google_forms' + +# # TODO: Uncomment only the section relevant to the specified renewal_survey_backend. + +# # Renewal survey settings: 'google_forms' backend. +# renewal_survey_google_forms_service_account_credentials_file_path: '/path/to/google-service-account-key.json' +# renewal_survey_google_forms_survey_data_file_path: '/path/to/google-forms-survey-data.json' +# renewal_survey_google_forms_survey_data_cache_key: 'renewal_survey_google_forms_survey_data' + +# # Renewal survey settings: 'permissive' backend. +# # N/A: No applicable settings. + +#------------------------------------------------------------------------------ +# REST API settings +#------------------------------------------------------------------------------ + # If true, bypass all checks at job submission time. allow_all_jobs: false +#------------------------------------------------------------------------------ +# Sentry settings +#------------------------------------------------------------------------------ + # The URL of the Sentry instance to send errors to. sentry_dsn: "" ############################################################################### -# staging_settings +# Deployment-specific settings: staging, production, development ############################################################################### +# TODO: Uncomment the section pertaining to the current deployment type. + +#------------------------------------------------------------------------------ +# staging_settings +#------------------------------------------------------------------------------ + # # The type of deployment ('dev', 'prod', 'staging'). # deployment: staging @@ -182,6 +266,7 @@ sentry_dsn: "" # djangooperator: root # # Whether to run the Django application in DEBUG mode. +# # Note: Use uppercase True/False so that Python interprets this as a boolean. # debug: True # # The path to the parent directory containing the Git repository. @@ -224,9 +309,9 @@ sentry_dsn: "" # # TODO: Set these addresses to yours. # request_approval_cc_list: [] -############################################################################### +#------------------------------------------------------------------------------ # prod_settings -############################################################################### +#------------------------------------------------------------------------------ # # The type of deployment ('dev', 'prod', 'staging'). # deployment: prod @@ -235,6 +320,7 @@ sentry_dsn: "" # djangooperator: meli # # Whether to run the Django application in DEBUG mode. +# # Note: Use uppercase True/False so that Python interprets this as a boolean. # debug: False # # The path to the parent directory containing the Git repository. @@ -280,9 +366,9 @@ sentry_dsn: "" # # TODO: Set these addresses to yours. # request_approval_cc_list: [] -############################################################################### +#------------------------------------------------------------------------------ # dev_settings -############################################################################### +#------------------------------------------------------------------------------ # # The type of deployment ('dev', 'prod', 'staging'). # deployment: dev @@ -291,6 +377,7 @@ sentry_dsn: "" # djangooperator: vagrant # # Whether to run the Django application in DEBUG mode. +# # Note: Use uppercase True/False so that Python interprets this as a boolean. # debug: True # # The path to the parent directory containing the Git repository. diff --git a/bootstrap/ansible/settings_template.tmpl b/bootstrap/ansible/settings_template.tmpl index 67f0bd938..35e988ba9 100644 --- a/bootstrap/ansible/settings_template.tmpl +++ b/bootstrap/ansible/settings_template.tmpl @@ -75,6 +75,23 @@ EXTRA_EXTRA_APPS = [] # Extra middleware to be included. EXTRA_EXTRA_MIDDLEWARE = [] +#------------------------------------------------------------------------------ +# Django Cache settings +#------------------------------------------------------------------------------ + +CACHES = { + 'default': { + 'BACKEND': 'django_redis.cache.RedisCache', + 'LOCATION': 'redis://{{ redis_host }}:6379/0', + 'OPTIONS': { + 'CLIENT_CLASS': 'django_redis.client.DefaultClient', + 'DB': 0, + 'PASSWORD': '{{ redis_passwd }}', + }, + 'TIMEOUT': 24 * 60 * 60, # One day in seconds + } +} + #------------------------------------------------------------------------------ # File Storage Settings #------------------------------------------------------------------------------ @@ -121,6 +138,32 @@ ORACLE_BILLING_DB = { } {% endif %} +#------------------------------------------------------------------------------ +# Renewal Survey settings +#------------------------------------------------------------------------------ +{% if renewal_survey_backend == 'google_forms' %} +# Use Google Forms as the backend for renewal surveys. +RENEWAL_SURVEY = { + 'backend': 'coldfront.core.project.utils_.renewal_survey.backends.google_forms.GoogleFormsRenewalSurveyBackend', + 'details': { + # The path to a file containing credentials for the Google service + # account that has access to survey responses. + 'credentials_file_path': '{{ renewal_survey_google_forms_service_account_credentials_file_path }}', + # The path to a file on local disk that contains data about each survey. + 'survey_data_file_path': '{{ renewal_survey_google_forms_survey_data_file_path }}', + # The key to cache survey data from the above file under. + 'survey_data_cache_key': '{{ renewal_survey_google_forms_survey_data_cache_key }}', + }, +} +{% else %} +# Permit renewal requests to be made without a survey response. +RENEWAL_SURVEY = { + 'backend': 'coldfront.core.project.utils_.renewal_survey.backends.permissive.PermissiveRenewalSurveyBackend', + 'details': {}, +} + +{% endif %} + #------------------------------------------------------------------------------ # SSL settings #------------------------------------------------------------------------------ @@ -246,6 +289,7 @@ FLAGS = { 'SERVICE_UNITS_PURCHASABLE': [{'condition': 'boolean', 'value': {{ flag_brc_enabled }}}], 'SSO_ENABLED': [{'condition': 'boolean', 'value': {{ flag_sso_enabled }}}], 'MOU_GENERATION_ENABLED': [{'condition': 'boolean', 'value': {{ flag_mou_generation_enabled }}}], + 'RENEWAL_SURVEY_ENABLED': [{'condition': 'boolean', 'value': {{ flag_renewal_survey_enabled }}}], } # Enforce that boolean flags are consistent with each other. diff --git a/bootstrap/development/docker/README.md b/bootstrap/development/docker/README.md index eaf6ee622..0640a2a83 100644 --- a/bootstrap/development/docker/README.md +++ b/bootstrap/development/docker/README.md @@ -51,7 +51,7 @@ Note that these steps must be run from the root directory of the repo. ```bash export DOCKER_PROJECT_NAME=brc-dev - docker-compose \ + docker compose \ -f bootstrap/development/docker/docker-compose.yml \ -p $DOCKER_PROJECT_NAME \ up @@ -69,13 +69,20 @@ Note that these steps must be run from the root directory of the repo. Notes: - This step may be run multiple times. -8. Retrieve a PostgreSQL database dump file that will be provided for you. Place it in the root directory of the repo. Load it into your instance. You must provide the name of your Docker project. Note that this may take several minutes. +8. Retrieve a PostgreSQL database dump file that will be provided for you. Place it in the root directory of the repo. Load it into your instance. You must provide the name of your Docker project. ```bash export RELATIVE_CONTAINER_DUMP_FILE_PATH=YYYY_MM_DD-HH-MM.dump sh bootstrap/development/docker/scripts/docker_load_database_backup.sh $DOCKER_PROJECT_NAME $RELATIVE_CONTAINER_DUMP_FILE_PATH ``` + Notes: + - This may take several minutes. + - The following error may appear in the output, but is not an issue: + ``` + ERROR: role "postgres" already exists + ``` + 9. At this point, the web service should be functioning. Navigate to it from the browser at "http://localhost:WEB_PORT", where `WEB_PORT` is the one defined above. 10. After authenticating for the first time, grant your user administrator privileges in Django: @@ -83,7 +90,7 @@ Note that these steps must be run from the root directory of the repo. - Enter into the application shell container: ```bash - docker-compose -p $DOCKER_PROJECT_NAME exec app-shell bash + docker compose -p $DOCKER_PROJECT_NAME exec app-shell bash ``` - From within the container, start a Django shell: diff --git a/bootstrap/development/docker/config/docker_defaults.yml b/bootstrap/development/docker/config/docker_defaults.yml index 6adcea52d..deecfd7a0 100644 --- a/bootstrap/development/docker/config/docker_defaults.yml +++ b/bootstrap/development/docker/config/docker_defaults.yml @@ -19,3 +19,5 @@ from_email: developer@localhost admin_email: developer@localhost email_admin_list: ['developer@localhost'] request_approval_cc_list: ['developer@localhost'] + +renewal_survey_backend: 'permissive' diff --git a/bootstrap/development/docker/images/app-base.Dockerfile b/bootstrap/development/docker/images/app-base.Dockerfile index 42cbb6ea3..912c4d6ea 100644 --- a/bootstrap/development/docker/images/app-base.Dockerfile +++ b/bootstrap/development/docker/images/app-base.Dockerfile @@ -1,12 +1,13 @@ -FROM ubuntu:latest +FROM coldfront-os -RUN apt-get update && \ - apt-get install -y python3 python3-dev python3-pip && \ - apt-get install -y libfaketime && \ - # Necessary for mod-wsgi requirement - apt-get install -y apache2-dev +WORKDIR /var/www/coldfront_app/coldfront + +RUN python3 -m venv /var/www/coldfront_app/venv COPY requirements.txt . -RUN python3 -m pip install -r requirements.txt +# Pin setuptools to avoid ImportError. Source: https://stackoverflow.com/a/78387663 +RUN /var/www/coldfront_app/venv/bin/pip install --upgrade pip wheel && \ + /var/www/coldfront_app/venv/bin/pip install setuptools==68.2.2 && \ + /var/www/coldfront_app/venv/bin/pip install -r requirements.txt -WORKDIR /var/www/coldfront_app/coldfront +ENV PATH="/var/www/coldfront_app/venv/bin:$PATH" diff --git a/bootstrap/development/docker/images/app-config.Dockerfile b/bootstrap/development/docker/images/app-config.Dockerfile index e6bf94423..2b7789eea 100644 --- a/bootstrap/development/docker/images/app-config.Dockerfile +++ b/bootstrap/development/docker/images/app-config.Dockerfile @@ -1,8 +1,10 @@ -FROM ubuntu:latest +FROM coldfront-os -RUN apt-get update && \ - apt-get install -y python3 python3-dev python3-pip +WORKDIR /app -RUN pip3 install jinja2 pyyaml +RUN python3 -m venv /app/venv -WORKDIR /app +RUN /app/venv/bin/pip install --upgrade pip setuptools wheel && \ + /app/venv/bin/pip install jinja2 pyyaml + +ENV PATH="/app/venv/bin:$PATH" diff --git a/bootstrap/development/docker/images/db-postgres-shell.Dockerfile b/bootstrap/development/docker/images/db-postgres-shell.Dockerfile index 8e8879f82..3e22a5b0a 100644 --- a/bootstrap/development/docker/images/db-postgres-shell.Dockerfile +++ b/bootstrap/development/docker/images/db-postgres-shell.Dockerfile @@ -1,12 +1,9 @@ -FROM ubuntu:latest +FROM coldfront-app-base -RUN apt-get update && \ - apt-get install -y gnupg lsb-release wget +RUN dnf update -y && \ + dnf install -y gnupg wget -RUN sh -c 'echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' && \ - wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - && \ - apt-get update && \ - apt-get -y install postgresql-client-15 +RUN dnf module install -y postgresql:15 WORKDIR /var/www/coldfront_app/coldfront diff --git a/bootstrap/development/docker/images/os.Dockerfile b/bootstrap/development/docker/images/os.Dockerfile new file mode 100644 index 000000000..bf1e9e4ad --- /dev/null +++ b/bootstrap/development/docker/images/os.Dockerfile @@ -0,0 +1,31 @@ +FROM rockylinux/rockylinux:8.8 + +RUN dnf update -y && \ + dnf install -y \ + gcc \ + make \ + wget \ + openssl-devel \ + bzip2-devel \ + httpd-devel \ + libffi-devel \ + zlib-devel \ + readline-devel \ + redhat-rpm-config \ + sqlite-devel + +RUN mkdir -p /usr/src/python310 && \ + cd /usr/src/python310 && \ + wget https://www.python.org/ftp/python/3.10.14/Python-3.10.14.tgz && \ + tar -xzvf Python-3.10.14.tgz && \ + cd Python-3.10.14 && \ + ./configure --enable-optimizations --enable-shared && \ + make && \ + make install + +RUN rm -rf /usr/src/python310 + +RUN echo "/usr/local/lib" > /etc/ld.so.conf.d/local.conf && \ + ldconfig + +# TODO: libfaketime diff --git a/bootstrap/development/docker/scripts/build_images.sh b/bootstrap/development/docker/scripts/build_images.sh index 8c4bcccb4..6aca0e44a 100755 --- a/bootstrap/development/docker/scripts/build_images.sh +++ b/bootstrap/development/docker/scripts/build_images.sh @@ -1,5 +1,6 @@ #!/bin/bash +docker build -f bootstrap/development/docker/images/os.Dockerfile -t coldfront-os . docker build -f bootstrap/development/docker/images/app-config.Dockerfile -t coldfront-app-config bootstrap/development/docker docker build -f bootstrap/development/docker/images/app-base.Dockerfile -t coldfront-app-base . docker build -f bootstrap/development/docker/images/app-shell.Dockerfile -t coldfront-app-shell . diff --git a/bootstrap/development/docker/scripts/create_env_file.sh b/bootstrap/development/docker/scripts/create_env_file.sh index a34caac74..5f4a14221 100755 --- a/bootstrap/development/docker/scripts/create_env_file.sh +++ b/bootstrap/development/docker/scripts/create_env_file.sh @@ -24,4 +24,3 @@ fi echo "DB_NAME=$DB_NAME" > $ENV_FILE_PATH echo "WEB_PORT=$PORT" >> $ENV_FILE_PATH - diff --git a/bootstrap/development/docker/scripts/docker_generate_secrets.sh b/bootstrap/development/docker/scripts/docker_generate_secrets.sh index 231a5a933..1630f4954 100755 --- a/bootstrap/development/docker/scripts/docker_generate_secrets.sh +++ b/bootstrap/development/docker/scripts/docker_generate_secrets.sh @@ -8,8 +8,11 @@ else wd=$PWD fi +# Do not mount directly onto /app, since the venv is located there and would be +# wiped out. docker run -it \ - -v $wd/bootstrap/development/docker:/app \ + -v $wd/bootstrap/development/docker/config:/app/config \ + -v $wd/bootstrap/development/docker/scripts:/app/scripts \ + -v $wd/bootstrap/development/docker/secrets:/app/secrets \ coldfront-app-config:latest \ python3 scripts/generate_secrets.py - diff --git a/bootstrap/development/docker/scripts/docker_generate_settings.sh b/bootstrap/development/docker/scripts/docker_generate_settings.sh index 943becb55..c89f9946c 100755 --- a/bootstrap/development/docker/scripts/docker_generate_settings.sh +++ b/bootstrap/development/docker/scripts/docker_generate_settings.sh @@ -22,8 +22,11 @@ cp coldfront/config/local_strings.py.sample coldfront/config/local_strings.py cp bootstrap/ansible/main.copyme bootstrap/development/docker/config/main.yml # Re-generate the Django development settings file. +# Do not mount directly onto /app, since the venv is located there and would be +# wiped out. (docker run -it \ -v $wd/bootstrap/ansible/settings_template.tmpl:/tmp/settings_template.tmpl \ - -v $wd/bootstrap/development/docker:/app \ + -v $wd/bootstrap/development/docker/config:/app/config \ + -v $wd/bootstrap/development/docker/scripts:/app/scripts \ coldfront-app-config:latest \ python3 scripts/generate_django_settings_file.py $DEPLOYMENT) 2>/dev/null > coldfront/config/dev_settings.py diff --git a/bootstrap/development/docker/scripts/docker_load_database_backup.sh b/bootstrap/development/docker/scripts/docker_load_database_backup.sh index 6ad9d6753..3a65248be 100755 --- a/bootstrap/development/docker/scripts/docker_load_database_backup.sh +++ b/bootstrap/development/docker/scripts/docker_load_database_backup.sh @@ -4,11 +4,11 @@ PROJECT_NAME=$1 RELATIVE_CONTAINER_DUMP_FILE_PATH=$2 # TODO: There may be other services in the future. -docker-compose -p $PROJECT_NAME stop web +docker compose -p $PROJECT_NAME stop web -docker-compose \ +docker compose \ -p $PROJECT_NAME \ exec db-postgres-shell \ bash -c "bootstrap/development/docker/scripts/load_database_backup.sh $RELATIVE_CONTAINER_DUMP_FILE_PATH" -docker-compose -p $PROJECT_NAME start web +docker compose -p $PROJECT_NAME start web diff --git a/bootstrap/development/docker/scripts/docker_run_django_scripts.sh b/bootstrap/development/docker/scripts/docker_run_django_scripts.sh index 7709ff13a..fbaa776f4 100755 --- a/bootstrap/development/docker/scripts/docker_run_django_scripts.sh +++ b/bootstrap/development/docker/scripts/docker_run_django_scripts.sh @@ -2,4 +2,4 @@ PROJECT_NAME=$1 -docker-compose -p $PROJECT_NAME exec app-shell bash -c "bootstrap/development/docker/scripts/run_django_scripts.sh" +docker compose -p $PROJECT_NAME exec app-shell bash -c "bootstrap/development/docker/scripts/run_django_scripts.sh" diff --git a/coldfront/config/local_settings.py.sample b/coldfront/config/local_settings.py.sample index 9bf8fb18f..8eb6cac5f 100644 --- a/coldfront/config/local_settings.py.sample +++ b/coldfront/config/local_settings.py.sample @@ -439,8 +439,7 @@ SESAME_MAX_AGE = 300 # Data import settings #------------------------------------------------------------------------------ -# The credentials needed to read from Google Sheets. -GOOGLE_OAUTH2_KEY_FILE = "/tmp/credentials.json" + # ----------------------------------------------------------------------------- # Miscellaneous settings diff --git a/coldfront/config/test_settings.py.sample b/coldfront/config/test_settings.py.sample index 431d990f6..98b8c7677 100644 --- a/coldfront/config/test_settings.py.sample +++ b/coldfront/config/test_settings.py.sample @@ -66,6 +66,16 @@ EXTRA_EXTRA_APPS = [] # Extra middleware to be included. EXTRA_EXTRA_MIDDLEWARE = [] +#------------------------------------------------------------------------------ +# Django Cache settings +#------------------------------------------------------------------------------ + +CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', + } +} + #------------------------------------------------------------------------------ # File Storage Settings #------------------------------------------------------------------------------ @@ -90,6 +100,16 @@ FILE_STORAGE = { }, } +#------------------------------------------------------------------------------ +# Renewal Survey settings +#------------------------------------------------------------------------------ + +# Permit renewal requests to be made without a survey response. +RENEWAL_SURVEY = { + 'backend': 'coldfront.core.project.utils_.renewal_survey.backends.permissive.PermissiveRenewalSurveyBackend', + 'details': {}, +} + #------------------------------------------------------------------------------ # SSL settings #------------------------------------------------------------------------------ @@ -128,4 +148,5 @@ FLAGS = { 'SERVICE_UNITS_PURCHASABLE': [{'condition': 'boolean', 'value': True}], 'SSO_ENABLED': [{'condition': 'boolean', 'value': False}], 'MOU_GENERATION_ENABLED': [{'condition': 'boolean', 'value': False}], + 'RENEWAL_SURVEY_ENABLED': [{'condition': 'boolean', 'value': True}], } diff --git a/coldfront/core/allocation/forms.py b/coldfront/core/allocation/forms.py index 4247e2ecb..f5f4b65d2 100644 --- a/coldfront/core/allocation/forms.py +++ b/coldfront/core/allocation/forms.py @@ -281,13 +281,16 @@ class AllocationPeriodChoiceField(forms.ModelChoiceField): def __init__(self, *args, **kwargs): self.computing_allowance = kwargs.pop('computing_allowance', None) + if (self.computing_allowance is not None and + not isinstance(self.computing_allowance, ComputingAllowance)): + self.computing_allowance = ComputingAllowance( + self.computing_allowance) self.interface = ComputingAllowanceInterface() super().__init__(*args, **kwargs) def label_from_instance(self, obj): - computing_allowance = ComputingAllowance(self.computing_allowance) num_service_units = self.allocation_value(obj) - if computing_allowance.are_service_units_prorated(): + if self.computing_allowance.are_service_units_prorated(): num_service_units = prorated_allocation_amount( num_service_units, utc_now_offset_aware(), obj) return ( @@ -297,7 +300,7 @@ def label_from_instance(self, obj): def allocation_value(self, obj): """Return the allocation value (Decimal) to use based on the allocation type and the AllocationPeriod.""" - allowance_name = self.computing_allowance.name + allowance_name = self.computing_allowance.get_name() if flag_enabled('BRC_ONLY'): assert allowance_name in self._allowances_with_periods_brc() return Decimal( diff --git a/coldfront/core/allocation/utils.py b/coldfront/core/allocation/utils.py index ca6e44ee2..4566ea253 100644 --- a/coldfront/core/allocation/utils.py +++ b/coldfront/core/allocation/utils.py @@ -237,6 +237,30 @@ def prorated_allocation_amount(amount, dt, allocation_period): return Decimal(f'{math.floor(amount):.2f}') +def calculate_service_units_to_allocate(computing_allowance, + request_time, allocation_period=None): + """Return the number of service units to allocate to a new project + request or allowance renewal request with the given + ComputingAllowance, if it were to be made at the given datetime. If + the request is associated with an AllocationPeriod, use it to + determine the number. Prorate as needed.""" + kwargs = {} + if allocation_period is not None: + kwargs['is_timed'] = True + kwargs['allocation_period'] = allocation_period + + computing_allowance_interface = ComputingAllowanceInterface() + num_service_units = Decimal( + computing_allowance_interface.service_units_from_name( + computing_allowance.get_name(), **kwargs)) + + if computing_allowance.are_service_units_prorated(): + num_service_units = prorated_allocation_amount( + num_service_units, request_time, allocation_period) + + return num_service_units + + def review_cluster_access_requests_url(): domain = settings.CENTER_BASE_URL view = reverse('allocation-cluster-account-request-list') diff --git a/coldfront/core/allocation/views_/secure_dir_views.py b/coldfront/core/allocation/views_/secure_dir_views.py index 4856284b8..7f28991dd 100644 --- a/coldfront/core/allocation/views_/secure_dir_views.py +++ b/coldfront/core/allocation/views_/secure_dir_views.py @@ -978,6 +978,9 @@ def done(self, form_list, **kwargs): @staticmethod def condition_dict(): + """Return a mapping from a string index `i` into FORMS + (zero-indexed) to a function determining whether FORMS[int(i)] + should be included.""" view = SecureDirRequestWizard return { '1': view.show_rdm_consultation_form_condition diff --git a/coldfront/core/project/forms_/new_project_forms/request_forms.py b/coldfront/core/project/forms_/new_project_forms/request_forms.py index 7029299dd..dc862c54e 100644 --- a/coldfront/core/project/forms_/new_project_forms/request_forms.py +++ b/coldfront/core/project/forms_/new_project_forms/request_forms.py @@ -51,9 +51,13 @@ class SavioProjectAllocationPeriodForm(forms.Form): def __init__(self, *args, **kwargs): computing_allowance = kwargs.pop('computing_allowance', None) super().__init__(*args, **kwargs) - display_timezone = pytz.timezone(settings.DISPLAY_TIME_ZONE) - queryset = self.allocation_period_choices( - computing_allowance, utc_now_offset_aware(), display_timezone) + if computing_allowance is not None: + computing_allowance = ComputingAllowance(computing_allowance) + display_timezone = pytz.timezone(settings.DISPLAY_TIME_ZONE) + queryset = self.allocation_period_choices( + computing_allowance, utc_now_offset_aware(), display_timezone) + else: + queryset = AllocationPeriod.objects.none() self.fields['allocation_period'] = AllocationPeriodChoiceField( computing_allowance=computing_allowance, label='Allocation Period', @@ -87,32 +91,29 @@ def allocation_period_choices(self, computing_allowance, utc_dt, def _allocation_period_choices_brc(self, computing_allowance, date, f, order_by): - """TODO""" - allowance_name = computing_allowance.name + allowance_name = computing_allowance.get_name() if allowance_name in (BRCAllowances.FCA, BRCAllowances.PCA): return self._allocation_period_choices_allowance_year( - date, f, order_by) + computing_allowance, date, f, order_by) elif allowance_name == BRCAllowances.ICA: num_days = self.NUM_DAYS_BEFORE_ICA f = f & Q(start_date__lte=date + timedelta(days=num_days)) - f = f & ( - Q(name__startswith='Fall Semester') | - Q(name__startswith='Spring Semester') | - Q(name__startswith='Summer Sessions')) + allowance_periods_q = computing_allowance.get_period_filters() + if allowance_periods_q is not None: + f = f & allowance_periods_q return AllocationPeriod.objects.filter(f).order_by(*order_by) return AllocationPeriod.objects.none() def _allocation_period_choices_lrc(self, computing_allowance, date, f, order_by): - """TODO""" - allowance_name = computing_allowance.name + allowance_name = computing_allowance.get_name() if allowance_name == LRCAllowances.PCA: return self._allocation_period_choices_allowance_year( - date, f, order_by) + computing_allowance, date, f, order_by) return AllocationPeriod.objects.none() - def _allocation_period_choices_allowance_year(self, date, f, order_by): - """TODO""" + def _allocation_period_choices_allowance_year(self, computing_allowance, + date, f, order_by): if flag_enabled('ALLOCATION_RENEWAL_FOR_NEXT_PERIOD_REQUESTABLE'): # If projects for the next period may be requested, include it. started_before_date = ( @@ -126,7 +127,9 @@ def _allocation_period_choices_allowance_year(self, date, f, order_by): # Otherwise, include only the current period. started_before_date = date f = f & Q(start_date__lte=started_before_date) - f = f & Q(name__startswith='Allowance Year') + allowance_periods_q = computing_allowance.get_period_filters() + if allowance_periods_q is not None: + f = f & allowance_periods_q return AllocationPeriod.objects.filter(f).order_by(*order_by) diff --git a/coldfront/core/project/forms_/renewal_forms/request_forms.py b/coldfront/core/project/forms_/renewal_forms/request_forms.py index f6dcba8e0..3f656fbfa 100644 --- a/coldfront/core/project/forms_/renewal_forms/request_forms.py +++ b/coldfront/core/project/forms_/renewal_forms/request_forms.py @@ -1,3 +1,5 @@ +import logging + from coldfront.core.allocation.models import AllocationPeriod from coldfront.core.allocation.models import AllocationRenewalRequest from coldfront.core.project.forms import DisabledChoicesSelectWidget @@ -12,11 +14,16 @@ from coldfront.core.project.utils_.renewal_utils import pis_with_renewal_requests_pks from coldfront.core.resource.utils_.allowance_utils.computing_allowance import ComputingAllowance from coldfront.core.resource.utils_.allowance_utils.interface import ComputingAllowanceInterface +from coldfront.core.project.utils_.renewal_survey import is_renewal_survey_completed from flags.state import flag_enabled from django import forms from django.utils.safestring import mark_safe from django.core.validators import MinLengthValidator +from django.core.exceptions import ValidationError + + +logger = logging.getLogger(__name__) class ProjectRenewalPIChoiceField(forms.ModelChoiceField): @@ -194,7 +201,51 @@ def __init__(self, *args, **kwargs): self.fields['project'].queryset = Project.objects.filter( **_filter).exclude(**exclude).order_by('name') + class ProjectRenewalSurveyForm(forms.Form): + def __init__(self, *args, **kwargs): + self.project_name = kwargs.pop('project_name', None) + self.pi_username = kwargs.pop('pi_username', None) + self.allocation_period_name = kwargs.pop('allocation_period_name', None) + super().__init__(*args, **kwargs) + + self.fields['was_survey_completed'] = forms.BooleanField( + label=('I have completed the survey and did NOT edit the pre-filled' + ' fields at the end of the survey.'), + initial=False, + required=True, + validators=[self.validate_survey_completed] + ) + + def validate_survey_completed(self, value): + """Raise ValidationError if no renewal survey was completed for + the specified PI and Project under the specified + AllocationPeriod. + + If completion cannot be determined (e.g., due to an error in the + survey backend), do not raise an error (such that the user may + proceed). + """ + try: + is_survey_completed = is_renewal_survey_completed( + self.allocation_period_name, self.project_name, + self.pi_username) + except Exception as e: + message = ( + f'Encountered an exception when determining whether a renewal ' + f'survey was completed for {self.allocation_period_name}, ' + f'{self.project_name}, {self.pi_username}. Allowing the user ' + f'to proceed to the next step. Details:\n{e}') + logger.exception(message) + return + + if not is_survey_completed: + raise ValidationError( + f'No response for {self.pi_username} and {self.project_name} ' + f'detected for {self.allocation_period_name}.') + + +class DeprecatedProjectRenewalSurveyForm(forms.Form): def __init__(self, *args, **kwargs): disable_fields = kwargs.pop('disable_fields', False) diff --git a/coldfront/core/project/management/commands/projects.py b/coldfront/core/project/management/commands/projects.py index dd441f982..d8c9eac7b 100644 --- a/coldfront/core/project/management/commands/projects.py +++ b/coldfront/core/project/management/commands/projects.py @@ -7,9 +7,14 @@ from flags.state import flag_enabled from coldfront.core.allocation.models import Allocation +from coldfront.core.allocation.models import AllocationPeriod from coldfront.core.allocation.models import AllocationAttribute from coldfront.core.allocation.models import AllocationAttributeType from coldfront.core.allocation.models import AllocationStatusChoice +from coldfront.core.allocation.models import AllocationRenewalRequest +from coldfront.core.allocation.models import AllocationRenewalRequestStatusChoice +from coldfront.core.allocation.utils import calculate_service_units_to_allocate +from coldfront.core.resource.utils_.allowance_utils.interface import ComputingAllowanceInterface from coldfront.core.project.models import Project from coldfront.core.project.models import ProjectStatusChoice from coldfront.core.project.models import ProjectUser @@ -18,8 +23,12 @@ from coldfront.core.project.utils import is_primary_cluster_project from coldfront.core.project.utils_.new_project_user_utils import NewProjectUserRunnerFactory from coldfront.core.project.utils_.new_project_user_utils import NewProjectUserSource +from coldfront.core.project.utils_.renewal_utils import AllocationRenewalApprovalRunner +from coldfront.core.project.utils_.renewal_utils import AllocationRenewalProcessingRunner +from coldfront.core.project.utils_.renewal_utils import set_allocation_renewal_request_eligibility from coldfront.core.resource.models import Resource from coldfront.core.resource.utils import get_primary_compute_resource_name +from coldfront.core.resource.utils_.allowance_utils.computing_allowance import ComputingAllowance from coldfront.core.statistics.models import ProjectTransaction from coldfront.core.utils.common import add_argparse_dry_run_argument from coldfront.core.utils.common import display_time_zone_current_date @@ -46,11 +55,15 @@ def add_arguments(self, parser): subparsers.required = True self._add_create_subparser(subparsers) + self._add_renew_subparser(subparsers) + def handle(self, *args, **options): """Call the handler for the provided subcommand.""" subcommand = options['subcommand'] if subcommand == 'create': self._handle_create(*args, **options) + elif subcommand == 'renew': + self._handle_renew(*args, **options) @staticmethod def _add_create_subparser(parsers): @@ -82,6 +95,32 @@ def _add_create_subparser(parsers): type=str) add_argparse_dry_run_argument(parser) + @staticmethod + def _add_renew_subparser(parsers): + """Add a subparser for the 'renew' subcommand.""" + parser = parsers.add_parser( + 'renew', + help='Renew a PI\'s allowance under a project.') + parser.add_argument( + 'name', help='The name of the project to renew under.', type=str) + parser.add_argument( + 'allocation_period', + help='The name of the AllocationPeriod to renew under.', + type=str) + parser.add_argument( + 'pi_username', + help=( + 'The username of the user whose allowance should be renewed. ' + 'The PI must be an active PI of the project.'), + type=str) + parser.add_argument( + 'requester_username', + help=( + 'The username of the user making the request. The requester ' + 'must be an active manager or PI of the project.'), + type=str) + add_argparse_dry_run_argument(parser) + @staticmethod def _create_project_with_compute_allocation_and_pis(project_name, compute_resource, @@ -180,6 +219,103 @@ def _handle_create(self, *args, **options): self.stdout.write(self.style.SUCCESS(message)) self.logger.info(message) + @staticmethod + def _renew_project(project, allocation_period, requester, pi, + computing_allowance, num_service_units): + """Renew the computing allowance of the given PI under the given + project for the given AllocationPeriod, as requested by the + given User and granting the given number of service units. + + 1. Create an AllocationRenewalRequest. + 2. Update the state of the request to prepare it for + approval. + 3. Approve the request. + 4. Process the request. + + Assumptions: + - The ComputingAllowance is renewable. + - (TODO: Temporary) The ComputingAllowance is not one per + PI. + - The PI is an active PI of the project. + - The requester is an active Manager or PI of the project. + - The allowance is being renewed under the same project + (i.e., there is no change in pooling preferences). + - The AllocationPeriod is current: it has started, and it + has not ended. + """ + pre_project = project + post_project = project + request_time = utc_now_offset_aware() + status = AllocationRenewalRequestStatusChoice.objects.get( + name='Under Review') + + with transaction.atomic(): + request = AllocationRenewalRequest.objects.create( + requester=requester, + pi=pi, + computing_allowance=computing_allowance.get_resource(), + allocation_period=allocation_period, + status=status, + pre_project=pre_project, + post_project=post_project, + request_time=request_time) + + eligibility_status = 'Approved' + eligibility_justification = '' + set_allocation_renewal_request_eligibility( + request, eligibility_status, eligibility_justification) + + # TODO: The command currently assumes that the period has already + # started. If allowing renewals for future periods: + # - Refactor and reuse existing logic for determining whether to + # run the processing runner. + # - Refactor ane reuse existing logic to filter out periods that + # are too far in the future. + + approval_runner = AllocationRenewalApprovalRunner( + request, num_service_units, email_strategy=DropEmailStrategy()) + approval_runner.run() + + request.refresh_from_db() + processing_runner = AllocationRenewalProcessingRunner( + request, num_service_units) + processing_runner.run() + + def _handle_renew(self, *args, **options): + """Handle the 'renew' subcommand.""" + cleaned_options = self._validate_renew_options(options) + + project_name = options['name'] + alloc_period_name = options['allocation_period'] + requester_str = options['requester_username'] + pi_str = options['pi_username'] + + message_template = ( + f'{{0}} the allocation for PI "{pi_str}" under Project ' + f'"{project_name}" for {alloc_period_name}, ' + f'requested by {requester_str}.') + if options['dry_run']: + message = message_template.format('Would renew') + self.stdout.write(self.style.WARNING(message)) + return + + try: + self._renew_project( + cleaned_options['project'], + cleaned_options['allocation_period'], + cleaned_options['requester'], + cleaned_options['pi'], + cleaned_options['computing_allowance'], + cleaned_options['num_service_units']) + except Exception as e: + message = message_template.format('Failed to renew') + self.stderr.write(self.style.ERROR(message)) + self.logger.exception(f'{message}\n{e}') + else: + message = message_template.format('Renewed') + self.stdout.write(self.style.SUCCESS(message)) + self.logger.info(message) + @staticmethod def _validate_create_options(options): """Validate the options provided to the 'create' subcommand. @@ -254,3 +390,124 @@ def _validate_create_options(options): 'compute_resource': compute_resource, 'pi_users': pi_users, } + + @staticmethod + def _validate_renew_options(options): + """Validate the options provided to the 'renew' subcommand. + Raise a subcommand if any are invalid or if they violate + business logic, else return a dict of the form: + { + 'project': Project, + 'requester': User, + 'pi': User, + 'allocation_period': AllocationPeriod, + 'computing_allowance': ComputingAllowance, + 'num_service_units': Decimal, + } + """ + project_name = options['name'].lower() + try: + project = Project.objects.get(name=project_name) + except Project.DoesNotExist: + raise CommandError( + f'A Project with name "{project_name}" does not exist.') + + computing_allowance_interface = ComputingAllowanceInterface() + computing_allowance = ComputingAllowance( + computing_allowance_interface.allowance_from_project(project)) + if not computing_allowance.is_renewable(): + raise CommandError( + f'Computing allowance "{computing_allowance.get_name()}" is ' + f'not renewable.') + + # TODO: There are some allowances which a PI may only have one of. + # Disallow renewals of these until business logic (in progress) is in + # place to enforce this constraint. + if computing_allowance.is_one_per_pi(): + raise CommandError( + 'Renewals of computing allowances that are limited per PI are ' + 'not currently supported by this command.') + + active_project_user_status = ProjectUserStatusChoice.objects.get( + name='Active') + manager_project_user_role = ProjectUserRoleChoice.objects.get( + name='Manager') + pi_project_user_role = ProjectUserRoleChoice.objects.get( + name='Principal Investigator') + + # The requester must be an "Active" "Manager" or "Principal + # Investigator" of the project. + requester_username = options['requester_username'] + try: + requester = User.objects.get(username=requester_username) + except User.DoesNotExist: + raise CommandError( + f'User with username "{requester_username}" does not exist.') + is_requester_valid = ProjectUser.objects.filter( + project=project, user=requester, + role__in=[manager_project_user_role, pi_project_user_role], + status=active_project_user_status).exists() + if not is_requester_valid: + raise CommandError( + f'Requester {requester.username} is not an active member of ' + f'the project "{project_name}".') + + # The PI must be an "Active" "Principal Investigator" of the project. + pi_username = options['pi_username'] + try: + pi = User.objects.get(username=pi_username) + except User.DoesNotExist: + raise CommandError( + f'User with username "{pi_username}" does not exist.') + is_pi_valid = ProjectUser.objects.filter( + project=project, user=pi, role=pi_project_user_role, + status=active_project_user_status).exists() + if not is_pi_valid: + raise CommandError( + f'{pi} is not an active PI of the project "{project_name}".') + + # The AllocationPeriod must be: + # (a) valid for the project's computing allowance, and + # (b) current. + allocation_period_name = options['allocation_period'] + try: + allocation_period = AllocationPeriod.objects.get( + name=allocation_period_name) + except AllocationPeriod.DoesNotExist: + raise CommandError( + f'AllocationPeriod "{allocation_period_name}" does not exist.') + + allowance_periods_q = computing_allowance.get_period_filters() + if not allowance_periods_q: + raise CommandError( + f'Unexpectedly found no AllocationPeriod filters for ' + f'"{computing_allowance.get_name()}".') + allocation_periods_for_allowance = AllocationPeriod.objects.filter( + allowance_periods_q) + try: + error = ( + f'"{allocation_period_name}" is not a valid AllocationPeriod ' + f'for computing allowance "{computing_allowance.get_name()}".') + assert allocation_period in allocation_periods_for_allowance, error + except AssertionError as e: + raise CommandError(e) + + try: + allocation_period.assert_started() + allocation_period.assert_not_ended() + except AssertionError as e: + raise CommandError(e) + + request_time = utc_now_offset_aware() + num_service_units = calculate_service_units_to_allocate( + computing_allowance, request_time, + allocation_period=allocation_period) + + return { + 'project': project, + 'requester': requester, + 'pi': pi, + 'allocation_period': allocation_period, + 'computing_allowance': computing_allowance, + 'num_service_units': num_service_units, + } diff --git a/coldfront/core/project/templates/project/project_renewal/project_renewal_survey.html b/coldfront/core/project/templates/project/project_renewal/project_renewal_survey.html index 2c798c88f..9f51b74e8 100644 --- a/coldfront/core/project/templates/project/project_renewal/project_renewal_survey.html +++ b/coldfront/core/project/templates/project/project_renewal/project_renewal_survey.html @@ -30,7 +30,29 @@

{{PROGRAM_NAME_SHORT}} Usage Survey

  • Requested Project: {{ requested_project }}
    • -

    Please respond to the following questions to provide us with more information about your project and computational needs.

    + +

    + Please fill out the survey below to provide us with more information about your project and computational needs. + + At the end of the survey, there are pre-filled fields needed for administrator purposes. Please do not edit them. + + If you edit them, your survey response may not be detected. +

    + +{% if renewal_survey_url %} + + Go to Survey + +{% else %} +

    + Survey unavailable. Please proceed to the next step. +

    +{% endif %} +

    +
    {% csrf_token %} diff --git a/coldfront/core/project/templates/project/project_renewal/request_survey_modal.html b/coldfront/core/project/templates/project/project_renewal/request_survey_modal.html index 9e6e03487..cedc8d674 100644 --- a/coldfront/core/project/templates/project/project_renewal/request_survey_modal.html +++ b/coldfront/core/project/templates/project/project_renewal/request_survey_modal.html @@ -17,7 +17,14 @@
    - {{ survey_form|crispy }} + + {% for question_and_answer in survey_response %} + + + + + {% endfor %} +
    {{ question_and_answer.0 }}{{ question_and_answer.1 }}
    diff --git a/coldfront/core/project/tests/test_commands/test_deactivate_ica_projects.py b/coldfront/core/project/tests/test_commands/test_deactivate_ica_projects.py index 936d3ed95..56e951177 100644 --- a/coldfront/core/project/tests/test_commands/test_deactivate_ica_projects.py +++ b/coldfront/core/project/tests/test_commands/test_deactivate_ica_projects.py @@ -10,7 +10,7 @@ AllocationUserAttributeUsage from coldfront.core.project.models import Project, ProjectStatusChoice, \ ProjectUserStatusChoice, ProjectUserRoleChoice, ProjectUser -from coldfront.core.project.utils import get_project_compute_allocation +from coldfront.core.allocation.utils import get_project_compute_allocation from coldfront.core.utils.common import utc_now_offset_aware from coldfront.core.project.tests.test_commands.test_service_units_base import TestSUBase diff --git a/coldfront/core/project/tests/test_commands/test_projects.py b/coldfront/core/project/tests/test_commands/test_projects.py new file mode 100644 index 000000000..9ee816220 --- /dev/null +++ b/coldfront/core/project/tests/test_commands/test_projects.py @@ -0,0 +1,471 @@ +from datetime import timedelta +from decimal import Decimal +from io import StringIO + +from django.conf import settings +from django.core.management import call_command +from django.core.management import CommandError +from django.contrib.auth.models import User + +from coldfront.api.statistics.utils import create_project_allocation + +from coldfront.core.allocation.models import AllocationPeriod +from coldfront.core.allocation.models import AllocationRenewalRequest +from coldfront.core.allocation.models import AllocationRenewalRequestStatusChoice +from coldfront.core.project.tests.test_commands.test_service_units_base import TestSUBase +from coldfront.core.project.models import Project +from coldfront.core.project.models import ProjectStatusChoice +from coldfront.core.project.models import ProjectUser +from coldfront.core.project.models import ProjectUserRoleChoice +from coldfront.core.project.models import ProjectUserStatusChoice +from coldfront.core.project.utils_.renewal_utils import get_current_allowance_year_period +from coldfront.core.resource.models import Resource +from coldfront.core.resource.models import ResourceAttributeType +from coldfront.core.resource.models import TimedResourceAttribute +from coldfront.core.resource.utils_.allowance_utils.constants import BRCAllowances +from coldfront.core.resource.utils_.allowance_utils.computing_allowance import ComputingAllowance +from coldfront.core.resource.utils_.allowance_utils.interface import ComputingAllowanceInterface +from coldfront.core.utils.common import display_time_zone_current_date +from coldfront.core.utils.tests.test_base import enable_deployment + + +class TestProjectsBase(TestSUBase): + """A base class for tests of the projects management command.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._command = ProjectsCommand() + + +class TestProjectsCreate(TestProjectsBase): + """A class for testing the 'create' subcommand of the 'projects' + management command.""" + + # TODO + pass + + +class TestProjectsRenew(TestProjectsBase): + """A class for testing the 'renew' subcommand of the 'projects' + management command.""" + + @enable_deployment('BRC') + def setUp(self): + super().setUp() + + self.create_test_user() + self.sign_user_access_agreement(self.user) + self.client.login(username=self.user.username, password=self.password) + + self._ica_computing_allowance = ComputingAllowance( + Resource.objects.get(name=BRCAllowances.ICA)) + computing_allowance_interface = ComputingAllowanceInterface() + project_name_prefix = computing_allowance_interface.code_from_name( + self._ica_computing_allowance.get_name()) + + # An arbitrary number of service units to grant to ICAs in these tests. + self._ica_num_service_units = Decimal('1000000.00') + + self._set_up_allocation_periods() + + # Create an inactive project and make self.user the PI + self.project_name = f'{project_name_prefix}project' + inactive_project_status = ProjectStatusChoice.objects.get( + name='Inactive') + inactive_project = Project.objects.create( + name=self.project_name, + title=self.project_name, + status=inactive_project_status) + + pi_role = ProjectUserRoleChoice.objects.get( + name='Principal Investigator') + active_project_user_status = ProjectUserStatusChoice.objects.get( + name='Active') + ProjectUser.objects.create( + project=inactive_project, + role=pi_role, + status=active_project_user_status, + user=self.user) + + accounting_allocation_objects = create_project_allocation( + inactive_project, settings.ALLOCATION_MIN) + self.service_units_attribute = \ + accounting_allocation_objects.allocation_attribute + + def _assert_project_inactive(self, project_name): + """Assert that a project has the Inactive status.""" + still_inactive_proj = Project.objects.get(name=project_name) + self.assertEqual( + still_inactive_proj.status, + ProjectStatusChoice.objects.get(name='Inactive')) + + def _set_up_allocation_periods(self): + """Create AllocationPeriods to potentially renew under.""" + # Delete existing ICA AllocationPeriods. + AllocationPeriod.objects.filter( + self._ica_computing_allowance.get_period_filters()).delete() + + today = display_time_zone_current_date() + year = today.year + + self.past_ica_period = AllocationPeriod.objects.create( + name=f'Spring Semester {year}', + start_date=today - timedelta(days=100), + end_date=today - timedelta(days=1)) + self.current_ica_period = AllocationPeriod.objects.create( + name=f'Summer Sessions {year}', + start_date=today - timedelta(days=50), + end_date=today + timedelta(days=50)) + self.future_ica_period = AllocationPeriod.objects.create( + name=f'Fall Semester {year + 1}', + start_date=today + timedelta(days=1), + end_date=today + timedelta(days=100)) + + ica_periods = ( + self.past_ica_period, + self.current_ica_period, + self.future_ica_period, + ) + for period in ica_periods: + self._set_service_units_to_be_allocated_for_period( + self._ica_computing_allowance, period, + self._ica_num_service_units) + + def _set_service_units_to_be_allocated_for_period(self, computing_allowance, + allocation_period, + num_service_units): + """Define the number of service units that should be granted to + as part of the given ComputingAllowance under the given + AllocationPeriod.""" + assert isinstance(computing_allowance, ComputingAllowance) + assert isinstance(allocation_period, AllocationPeriod) + assert isinstance(num_service_units, Decimal) + resource_attribute_type = ResourceAttributeType.objects.get( + name='Service Units') + TimedResourceAttribute.objects.update_or_create( + resource_attribute_type=resource_attribute_type, + resource=computing_allowance.get_resource(), + start_date=allocation_period.start_date, + end_date=allocation_period.end_date, + defaults={ + 'value': str(num_service_units), + }) + + @enable_deployment('BRC') + def test_dry_run(self): + """Test that the request would be successful but the dry run + ensures that the project is not updated.""" + project = Project.objects.get(name=self.project_name) + self.assertEqual(project.status, + ProjectStatusChoice.objects.get(name='Inactive')) + output, error = self._command.renew( + self.project_name, self.current_ica_period, self.user.username, + self.user.username, dry_run=True) + + self.assertFalse(error) + + self.assertIn('Would renew', output) + self.service_units_attribute.refresh_from_db() + self.assertEqual( + Decimal(self.service_units_attribute.value), + settings.ALLOCATION_MIN) + + @enable_deployment('BRC') + def test_success(self): + """Test that a successful request updates a project's status + to 'Active' and the service units to the correct amount.""" + project = Project.objects.get(name=self.project_name) + self.assertEqual( + project.status, + ProjectStatusChoice.objects.get(name='Inactive')) + + output, error = self._command.renew( + self.project_name, self.current_ica_period, self.user.username, + self.user.username) + + self.assertFalse(error) + now_active_proj = Project.objects.get(name=self.project_name) + self.assertEqual( + now_active_proj.status, + ProjectStatusChoice.objects.get(name='Active')) + self.service_units_attribute.refresh_from_db() + self.assertEqual( + Decimal(self.service_units_attribute.value), + self._ica_num_service_units) + + request = AllocationRenewalRequest.objects.get( + requester=self.user, + pi=self.user, + pre_project=project) + self.assertEqual( + AllocationRenewalRequestStatusChoice.objects.get(name='Complete'), + request.status) + # TODO: Also test: + # That only a processing email is sent + + @enable_deployment('BRC') + def test_validate_project(self): + """Test that, if the project is invalid, the command raises an + error, and does not proceed.""" + project = Project.objects.get(name=self.project_name) + self.assertEqual( + project.status, ProjectStatusChoice.objects.get(name='Inactive')) + + with self.assertRaises(CommandError) as cm: + self._command.renew( + 'invalid project name', self.current_ica_period, + self.user.username, self.user.username) + self.assertIn('A Project with name', str(cm.exception)) + self._assert_project_inactive(self.project_name) + self.service_units_attribute.refresh_from_db() + self.assertEqual( + Decimal(self.service_units_attribute.value), + settings.ALLOCATION_MIN) + + @enable_deployment('BRC') + def test_validate_computing_allowance_non_renewable(self): + """Test that computing allowances that cannot be renewed fail + correctly (e.g., Recharge, Condo).""" + computing_allowance_interface = ComputingAllowanceInterface() + non_renewable_resources = [BRCAllowances.CO, BRCAllowances.RECHARGE] + for resource_name in non_renewable_resources: + computing_allowance = ComputingAllowance( + Resource.objects.get(name=resource_name)) + assert not computing_allowance.is_renewable() + project_name_prefix = computing_allowance_interface.code_from_name( + computing_allowance.get_name()) + project_name = project_name_prefix + 'testproject' + Project.objects.create( + name=project_name, + title=project_name, + status=ProjectStatusChoice.objects.get(name='Inactive')) + + with self.assertRaises(CommandError) as cm: + self._command.renew( + project_name, self.current_ica_period, + self.user.username, self.user.username) + self.assertIn('is not renewable', str(cm.exception)) + self._assert_project_inactive(project_name) + + # TODO: Retire this test case once support for these allowances has been + # added. + @enable_deployment('BRC') + def test_validate_computing_allowance_one_per_pi(self): + """Test that computing allowances which a PI may only have one + of fail correctly.""" + computing_allowance_interface = ComputingAllowanceInterface() + one_per_pi_resources = [BRCAllowances.FCA, BRCAllowances.PCA] + for resource_name in one_per_pi_resources: + computing_allowance = ComputingAllowance( + Resource.objects.get(name=resource_name)) + assert computing_allowance.is_one_per_pi() + project_name_prefix = computing_allowance_interface.code_from_name( + computing_allowance.get_name()) + project_name = project_name_prefix + 'testproject' + Project.objects.create( + name=project_name, + title=project_name, + status=ProjectStatusChoice.objects.get(name='Inactive')) + + with self.assertRaises(CommandError) as cm: + self._command.renew( + project_name, self.current_ica_period, + self.user.username, self.user.username) + self.assertIn('not currently supported', str(cm.exception)) + self._assert_project_inactive(project_name) + + @enable_deployment('BRC') + def test_validate_requester(self): + """Test that requesters who do not exist, or are not an active + manager/PI fail correctly.""" + project = Project.objects.get(name=self.project_name) + self.assertEqual( + project.status, ProjectStatusChoice.objects.get(name='Inactive')) + + # Requester who is not manager/PI is invalid. + invalid_requester = User.objects.create( + email='invalid@gmail.com', + first_name='invalid', + last_name='invalid', + username='invalid_requester' + ) + ProjectUser.objects.create( + project=project, + role=ProjectUserRoleChoice.objects.get(name='User'), + status=ProjectUserStatusChoice.objects.get(name='Active'), + user=invalid_requester) + + # Requester who is removed from project is invalid, even if Manager/PI + removed_requester = User.objects.create( + email='removed@gmail.com', + first_name='removed', + last_name='removed', + username='removed_requester' + ) + ProjectUser.objects.create( + project=project, + role=ProjectUserRoleChoice.objects.get(name='Manager'), + status=ProjectUserStatusChoice.objects.get(name='Removed'), + user=removed_requester) + + invalid_usernames = [invalid_requester.username, + removed_requester.username] + for invalid_username in invalid_usernames: + with self.assertRaises(CommandError) as cm: + self._command.renew( + project.name, self.current_ica_period, + self.user.username, invalid_username) + self.assertIn( + f'Requester {invalid_username} is not an active member', + str(cm.exception)) + self._assert_project_inactive(project.name) + self.service_units_attribute.refresh_from_db() + self.assertEqual( + Decimal(self.service_units_attribute.value), + settings.ALLOCATION_MIN) + + @enable_deployment('BRC') + def test_validate_pi(self): + """Test that PIs who do not exist or are not active fail + correctly.""" + project = Project.objects.get(name=self.project_name) + self.assertEqual( + project.status, ProjectStatusChoice.objects.get(name='Inactive')) + + nonexistent_pi_username = 'nonexistent_pi_username' + with self.assertRaises(CommandError) as cm: + self._command.renew( + project.name, self.current_ica_period, + nonexistent_pi_username, self.user.username) + self.assertIn( + f'User with username "{nonexistent_pi_username}" does not exist', + str(cm.exception)) + self._assert_project_inactive(project.name) + + # Active User (manager) who is on project but not PI + invalid_pi = User.objects.create( + email='manager@gmail.com', + first_name='manager', + last_name='manager', + username='invalid_pi' + ) + ProjectUser.objects.create( + project=project, + role=ProjectUserRoleChoice.objects.get(name='Manager'), + status=ProjectUserStatusChoice.objects.get(name='Active'), + user=invalid_pi) + + # Removed PI + removed_pi = User.objects.create( + email='removedPI@gmail.com', + first_name='removed', + last_name='removed', + username='removed_pi' + ) + ProjectUser.objects.create( + project=project, + role=ProjectUserRoleChoice.objects.get( + name='Principal Investigator'), + status=ProjectUserStatusChoice.objects.get(name='Removed'), + user=removed_pi) + invalid_pis = [invalid_pi.username, removed_pi.username] + for invalid_pi in invalid_pis: + with self.assertRaises(CommandError) as cm: + self._command.renew( + project.name, self.current_ica_period, + invalid_pi, self.user.username) + self.assertIn( + f'{invalid_pi} is not an active PI', + str(cm.exception)) + self._assert_project_inactive(project.name) + self.service_units_attribute.refresh_from_db() + self.assertEqual( + Decimal(self.service_units_attribute.value), + settings.ALLOCATION_MIN) + + @enable_deployment('BRC') + def test_validate_allocation_period(self): + """Test that AllocationPeriods which do not exist, are not valid + for the given computing allowance, or are not current fail + correctly.""" + project = Project.objects.get(name=self.project_name) + self.assertEqual( + project.status, ProjectStatusChoice.objects.get(name='Inactive')) + + nonexistent_alloc_period = 'I don\'t exist!' + with self.assertRaises(CommandError) as cm: + self._command.renew( + project.name, nonexistent_alloc_period, + self.user.username, self.user.username) + self.assertIn( + f'AllocationPeriod "{nonexistent_alloc_period}" does not exist.', + str(cm.exception)) + self._assert_project_inactive(project.name) + + # "Allowance Year" allocation periods are not for ICA projects + cur_allowance_year = get_current_allowance_year_period() + + with self.assertRaises(CommandError) as cm: + self._command.renew( + project.name, cur_allowance_year.name, + self.user.username, self.user.username) + self.assertIn( + f'"{cur_allowance_year.name}" is not a valid AllocationPeriod', + str(cm.exception)) + self._assert_project_inactive(project.name) + + # Ended allocation period + with self.assertRaises(CommandError) as cm: + self._command.renew( + project.name, self.past_ica_period, + self.user.username, self.user.username) + self.assertIn( + 'AllocationPeriod already ended', + str(cm.exception)) + self._assert_project_inactive(project.name) + + # Not started allocation period + with self.assertRaises(CommandError) as cm: + self._command.renew( + project.name, self.future_ica_period, + self.user.username, self.user.username) + self.assertIn( + 'AllocationPeriod does not start until', + str(cm.exception)) + self._assert_project_inactive(project.name) + + self.service_units_attribute.refresh_from_db() + self.assertEqual( + Decimal(self.service_units_attribute.value), + settings.ALLOCATION_MIN) + + +class ProjectsCommand(object): + """A wrapper class over the 'projects' management command.""" + + command_name = 'projects' + + def call_subcommand(self, name, *args): + """Call the subcommand with the given name and arguments. Return + output written to stdout and stderr.""" + out, err = StringIO(), StringIO() + args = [self.command_name, name, *args] + kwargs = {'stdout': out, 'stderr': err} + call_command(*args, **kwargs) + return out.getvalue(), err.getvalue() + + def renew(self, name, allocation_period, pi_username, requester_username, + **flags): + """Call the 'renew' subcommand with the given positional arguments.""" + args = [ + 'renew', name, allocation_period, pi_username, requester_username] + self._add_flags_to_args(args, **flags) + return self.call_subcommand(*args) + + @staticmethod + def _add_flags_to_args(args, **flags): + """Given a list of arguments to the command and a dict of flag + values, add the latter to the former.""" + for key in ('dry_run', 'ignore_invalid'): + if flags.get(key, False): + args.append(f'--{key}') diff --git a/coldfront/core/project/tests/test_forms/test_renewal_forms/test_project_renewal_survey_form.py b/coldfront/core/project/tests/test_forms/test_renewal_forms/test_project_renewal_survey_form.py new file mode 100644 index 000000000..f41a78c81 --- /dev/null +++ b/coldfront/core/project/tests/test_forms/test_renewal_forms/test_project_renewal_survey_form.py @@ -0,0 +1,109 @@ +from coldfront.core.project.forms_.renewal_forms.request_forms import ProjectRenewalSurveyForm +from coldfront.core.project.models import Project, ProjectStatusChoice, ProjectUser, ProjectUserRoleChoice, ProjectUserStatusChoice +from coldfront.core.project.utils_.renewal_utils import get_current_allowance_year_period +from coldfront.core.resource.utils_.allowance_utils.interface import ComputingAllowanceInterface +from coldfront.core.utils.tests.test_base import TestBase +from coldfront.core.project.utils_.renewal_survey.backends.base import BaseRenewalSurveyBackend + +from django.contrib.auth.models import User +from django.test import override_settings + + +@override_settings(RENEWAL_SURVEY = { + 'backend': 'coldfront.core.project.tests.test_forms.test_renewal_forms.test_project_renewal_survey_form.DummyRenewalSurveyBackend', + 'details': {}, +}) +class TestProjectRenewalSurveyForm(TestBase): + """A class for testing ProjectRenewalSurveyForm.""" + def setUp(self): + """Set up test data.""" + super().setUp() + computing_allowance_interface = ComputingAllowanceInterface() + computing_allowance = self.get_predominant_computing_allowance() + self._project_prefix = computing_allowance_interface.code_from_name( + computing_allowance.name) + + self.allocation_period = get_current_allowance_year_period() + + self.pi_role = ProjectUserRoleChoice.objects.get( + name='Principal Investigator') + self.active_project_user_status = ProjectUserStatusChoice.objects.get( + name='Active') + + def test_is_renewal_survey_completed_returns_false(self): + user = User.objects.create( + email='test_user@email.com', + first_name='Test', + last_name='User', + username='test_user') + + project_name = f'{self._project_prefix}project' + active_project_status = ProjectStatusChoice.objects.get(name='Active') + project = Project.objects.create( + name=project_name, + title=project_name, + status=active_project_status) + self.project_user = ProjectUser.objects.create( + project=project, + role=self.pi_role, + status=self.active_project_user_status, + user=user) + + form=ProjectRenewalSurveyForm( + project_name=project_name, + pi_username=user.username, + allocation_period_name=self.allocation_period.name, + data={'was_survey_completed': True} + ) + + self.assertFalse(form.is_valid()) + self.assertIn( + f'No response for {user.username} and {project_name} detected', + str(form.errors)) + + def test_is_renewal_survey_completed_returns_true(self): + user = User.objects.create( + email='test_user@email.com', + first_name='Test', + last_name='User', + username='test') + + project_name = f'{self._project_prefix}project2' + active_project_status = ProjectStatusChoice.objects.get(name='Active') + project = Project.objects.create( + name=project_name, + title=project_name, + status=active_project_status) + self.project_user = ProjectUser.objects.create( + project=project, + role=self.pi_role, + status=self.active_project_user_status, + user=user) + + form=ProjectRenewalSurveyForm( + project_name=project_name, + pi_username=user.username, + allocation_period_name=self.allocation_period.name, + data={'was_survey_completed': True} + ) + + self.assertTrue(form.is_valid()) + +class DummyRenewalSurveyBackend(BaseRenewalSurveyBackend): + """A backend similar to PermissiveRenewalSurveyBackend except + is_renewal_survey_completed returns True/False based on pi_username.""" + + def is_renewal_survey_completed(self, allocation_period_name, project_name, + pi_username): + """Return whether the pi_username is less than 5 characters long.""" + return len(pi_username) < 5 + + def get_renewal_survey_response(self, allocation_period_name, project_name, + pi_username): + """Return an empty list of responses.""" + return [] + + def get_renewal_survey_url(self, allocation_period_name, pi, project_name, + requester): + """Return an empty string.""" + return '' diff --git a/coldfront/core/project/tests/test_views/test_renewal_views/test_allocation_renewal_request_under_project_view.py b/coldfront/core/project/tests/test_views/test_renewal_views/test_allocation_renewal_request_under_project_view.py index f279e26c3..7cd5ac0b2 100644 --- a/coldfront/core/project/tests/test_views/test_renewal_views/test_allocation_renewal_request_under_project_view.py +++ b/coldfront/core/project/tests/test_views/test_renewal_views/test_allocation_renewal_request_under_project_view.py @@ -65,40 +65,7 @@ def _request_view_name(): """Return the name of the request view.""" return 'allocation_renewal_request_under_project_view' - @staticmethod - def _sample_renewal_survey_post_data(): - """Return a dict of renewal survey answers to be sent to the - view via a POST request.""" - return { - 'which_brc_services_used': 'srdc', - 'publications_supported_by_brc': 'N/A', - 'grants_supported_by_brc': 'N/A', - 'recruitment_or_retention_cases': 'N/A', - 'classes_being_taught': 'N/A', - 'brc_recommendation_rating': 6, - 'brc_recommendation_rating_reason': '', - 'how_brc_helped_bootstrap_computational_methods': 'N/A', - 'how_important_to_research_is_brc': 2, - 'do_you_use_mybrc': 'no', - 'mybrc_comments': '', - 'which_open_ondemand_apps_used': [ - 'jupyter_notebook', - 'vscode_server', - ], - 'brc_feedback': '', - 'colleague_suggestions': '', - 'indicate_topic_interests': [ - 'have_had_rdmp_event_or_consultation', - 'want_to_learn_more_and_have_rdm_consult', - ], - 'training_session_usefulness_of_computational_platforms_training': - 2, - 'training_session_usefulness_of_basic_savio_cluster': 4, - 'training_session_other_topics_of_interest': '', - } - - def _send_post_requests(self, allocation_period, project, pi_project_user, - include_renewal_survey=True): + def _send_post_requests(self, allocation_period, project, pi_project_user): """Send the necessary POST requests to the view for the given AllocationPeriod, Project, and PI ProjectUser. Optionally include sample renewal survey answers.""" @@ -119,15 +86,11 @@ def _send_post_requests(self, allocation_period, project, pi_project_user, } form_data.append(pi_selection_form_data) - if include_renewal_survey: - # While the renewal survey is only conditionally required, data may - # always be POSTed and accepted. - renewal_survey_step = '2' - renewal_survey_form_data = {} - for key, value in self._sample_renewal_survey_post_data().items(): - renewal_survey_form_data[f'{renewal_survey_step}-{key}'] = value - renewal_survey_form_data[current_step_key] = renewal_survey_step - form_data.append(renewal_survey_form_data) + renewal_survey_form_data = { + '2-was_survey_completed': True, + current_step_key: '2', + } + form_data.append(renewal_survey_form_data) review_and_submit_form_data = { '3-confirmation': True, @@ -154,8 +117,7 @@ def test_post_sets_request_request_time(self): allocation_period = get_current_allowance_year_period() self._send_post_requests( - allocation_period, project, pi_project_user, - include_renewal_survey=True) + allocation_period, project, pi_project_user) post_time = utc_now_offset_aware() @@ -163,40 +125,4 @@ def test_post_sets_request_request_time(self): self.assertEqual(requests.count(), 1) request = requests.first() self.assertTrue(pre_time <= request.request_time <= post_time) - self.assertTrue(request.renewal_survey_answers) - - # TODO: As of the time of this writing, only one period is selectable. - # Moreover, any future period will have the survey, so this test may be - # obsolete. - def test_renewal_survey_step_conditionally_required(self): - """Test that the renewal survey step is only required when the - selected AllocationPeriod is 'Allowance Year 2024 - 2025'. - - TODO: This period has been hard-coded for the short-term. Once a - longer-term solution without hard-coding has been applied, - update this test accordingly. - """ - project, pi_project_user = self._create_project_to_renew() - # The survey only needs to be provided for a particular period. - success_by_period_name = { - 'Allowance Year 2024 - 2025': False, - } - - for period_name, success_expected in success_by_period_name.items(): - allocation_period = AllocationPeriod.objects.get(name=period_name) - func = self._send_post_requests - args = [allocation_period, project, pi_project_user] - kwargs = {'include_renewal_survey': False} - if success_expected: - func(*args, **kwargs) - else: - with self.assertRaises(AssertionError) as _: - func(*args, **kwargs) - requests = AllocationRenewalRequest.objects.filter( - allocation_period=allocation_period, post_project=project) - if success_expected: - self.assertTrue(requests.count() == 1) - self.assertFalse(requests.first().renewal_survey_answers) - else: - self.assertFalse(requests.exists()) diff --git a/coldfront/core/project/tests/test_views/test_renewal_views/test_allocation_renewal_request_view.py b/coldfront/core/project/tests/test_views/test_renewal_views/test_allocation_renewal_request_view.py index c0d22297f..e8503950d 100644 --- a/coldfront/core/project/tests/test_views/test_renewal_views/test_allocation_renewal_request_view.py +++ b/coldfront/core/project/tests/test_views/test_renewal_views/test_allocation_renewal_request_view.py @@ -58,40 +58,7 @@ def _request_view_name(): """Return the name of the request view.""" return 'allocation_renewal_request_view' - @staticmethod - def _sample_renewal_survey_post_data(): - """Return a dict of renewal survey answers to be sent to the - view via a POST request.""" - return { - 'which_brc_services_used': 'srdc', - 'publications_supported_by_brc': 'N/A', - 'grants_supported_by_brc': 'N/A', - 'recruitment_or_retention_cases': 'N/A', - 'classes_being_taught': 'N/A', - 'brc_recommendation_rating': 6, - 'brc_recommendation_rating_reason': '', - 'how_brc_helped_bootstrap_computational_methods': 'N/A', - 'how_important_to_research_is_brc': 2, - 'do_you_use_mybrc': 'no', - 'mybrc_comments': '', - 'which_open_ondemand_apps_used': [ - 'jupyter_notebook', - 'vscode_server', - ], - 'brc_feedback': '', - 'colleague_suggestions': '', - 'indicate_topic_interests': [ - 'have_had_rdmp_event_or_consultation', - 'want_to_learn_more_and_have_rdm_consult', - ], - 'training_session_usefulness_of_computational_platforms_training': - 2, - 'training_session_usefulness_of_basic_savio_cluster': 4, - 'training_session_other_topics_of_interest': '', - } - - def _send_post_requests(self, allocation_period, pi_project_user, - include_renewal_survey=True): + def _send_post_requests(self, allocation_period, pi_project_user): """Send the necessary POST requests to the view for the given AllocationPeriod, Project, and PI ProjectUser. Optionally include sample renewal survey answers.""" @@ -118,15 +85,11 @@ def _send_post_requests(self, allocation_period, pi_project_user, } form_data.append(pooling_preference_form_data) - if include_renewal_survey: - # While the renewal survey is only conditionally required, data may - # always be POSTed and accepted. - renewal_survey_step = '6' - renewal_survey_form_data = {} - for key, value in self._sample_renewal_survey_post_data().items(): - renewal_survey_form_data[f'{renewal_survey_step}-{key}'] = value - renewal_survey_form_data[current_step_key] = renewal_survey_step - form_data.append(renewal_survey_form_data) + renewal_survey_form_data = { + '6-was_survey_completed': True, + current_step_key: '6', + } + form_data.append(renewal_survey_form_data) review_and_submit_form_data = { '7-confirmation': True, @@ -152,7 +115,7 @@ def test_post_sets_request_request_time(self): allocation_period = get_current_allowance_year_period() self._send_post_requests( - allocation_period, pi_project_user, include_renewal_survey=True) + allocation_period, pi_project_user) post_time = utc_now_offset_aware() @@ -160,40 +123,3 @@ def test_post_sets_request_request_time(self): self.assertEqual(requests.count(), 1) request = requests.first() self.assertTrue(pre_time <= request.request_time <= post_time) - self.assertTrue(request.renewal_survey_answers) - - # TODO: As of the time of this writing, only one period is selectable. - # Moreover, any future period will have the survey, so this test may be - # obsolete. - def test_renewal_survey_step_conditionally_required(self): - """Test that the renewal survey step is only required when the - selected AllocationPeriod is 'Allowance Year 2024 - 2025'. - - TODO: This period has been hard-coded for the short-term. Once a - longer-term solution without hard-coding has been applied, - update this test accordingly. - """ - project, pi_project_user = self._create_project_to_renew() - - # The survey only needs to be provided for a particular period. - success_by_period_name = { - 'Allowance Year 2024 - 2025': False, - } - - for period_name, success_expected in success_by_period_name.items(): - allocation_period = AllocationPeriod.objects.get(name=period_name) - func = self._send_post_requests - args = [allocation_period, pi_project_user] - kwargs = {'include_renewal_survey': False} - if success_expected: - func(*args, **kwargs) - else: - with self.assertRaises(AssertionError) as _: - func(*args, **kwargs) - requests = AllocationRenewalRequest.objects.filter( - allocation_period=allocation_period, post_project=project) - if success_expected: - self.assertTrue(requests.count() == 1) - self.assertFalse(requests.first().renewal_survey_answers) - else: - self.assertFalse(requests.exists()) diff --git a/coldfront/core/project/utils_/renewal_survey/__init__.py b/coldfront/core/project/utils_/renewal_survey/__init__.py new file mode 100644 index 000000000..55b7b5b08 --- /dev/null +++ b/coldfront/core/project/utils_/renewal_survey/__init__.py @@ -0,0 +1,93 @@ +from django.conf import settings +from django.utils.module_loading import import_string + +"""Methods relating to renewal survey backend.""" + + +__all__ = [ + 'get_backend', + 'is_renewal_survey_completed', + 'get_renewal_survey_response', + 'get_renewal_survey_url' +] + + +def get_backend(backend=None, **kwds): + klass = import_string(backend or settings.RENEWAL_SURVEY['backend']) + return klass(**kwds) + + +def is_renewal_survey_completed(allocation_period_name, project_name, + pi_username, backend=None): + """Return whether a renewal survey has been completed for the + given PI and project under the given AllocationPeriod. + + Parameters: + - allocation_period_name (str): the name of the AllocationPeriod + the allowance is being renewed under + - project_name (str): the name of the Project the allowance is + being renewed under + - pi_username (str): the username of the PI whose allowance is + being renewed + - backend (BaseRenewalSurveyBackend): an optional renewal survey + backend to use (default to the one defined in settings) + + Returns: + - bool + + Raises: + - Exception, if any errors occur. + """ + backend = backend or get_backend() + return backend.is_renewal_survey_completed( + allocation_period_name, project_name, pi_username) + + +def get_renewal_survey_response(allocation_period_name, project_name, pi_username, + backend=None): + """ Takes the identifying information for a response and finds the + specific survey response. Each question is then paired with its answer + in a tuple and the array of tuples in correct order are returned. If no + response is detected, return None. The format of the tuple: + ( question: string, answer: string ). + + - Inputs: + - Identifying information: + - allocation_period_name + - project_name + - pi_username + - backend: Users can inject a backend rather than default to + the backend determined in the settings. + + - Output: + - Array of Tuples (question, response) """ + backend = backend or get_backend() + return backend.get_renewal_survey_response( + allocation_period_name, project_name, pi_username) + + +def get_renewal_survey_url(allocation_period_name, pi, project_name, requester, + backend=None): + """ This function returns the unique link to a pre-filled form for the + user to fill out. + + - Inputs: + - allocation_period_name: Name of the allocation period the project + is being renewed under. This is pre-filled into the Allocation + Period question on the form. + - pi: The `User` object (from `django.contrib.auth.models`) for the + PI whose allowance is being renewed. The PI’s name and username + are pre-filled into the form. + - project_name: Name of the project being renewed. This is + pre-filled into the Project Name question on the form. + - requester: The `User` object (from `django.contrib.auth.models`) + for the user filling out the renewal request form. The + requester’s name and username are pre-filled into the form. + - backend: Users can inject a backend rather than default to + the backend determined in the settings. + + - Output: + - URL """ + backend = backend or get_backend() + return backend.get_renewal_survey_url( + allocation_period_name, pi, project_name, requester) diff --git a/coldfront/core/project/utils_/renewal_survey/backends/base.py b/coldfront/core/project/utils_/renewal_survey/backends/base.py new file mode 100644 index 000000000..24f66eec8 --- /dev/null +++ b/coldfront/core/project/utils_/renewal_survey/backends/base.py @@ -0,0 +1,69 @@ +from abc import ABC +from abc import abstractmethod + + +class BaseRenewalSurveyBackend(ABC): + """An interface for supporting a renewal survey hosted on any of a + number of backends.""" + + @abstractmethod + def is_renewal_survey_completed(self, allocation_period_name, project_name, + pi_username): + """Return whether a survey has been completed for the given + project and PI for the given allocation period. + + Parameters: + - allocation_period_name (str): the name of an AllocationPeriod + - project_name (str): the name of a Project + - pi_username (str): the username of a PI of the Project + + Returns: + - boolean + + Raises: + - Exception, if any errors occur. + """ + pass + + @abstractmethod + def get_renewal_survey_response(self, allocation_period_name, project_name, + pi_username): + """Return a list of (question, answer) tuples for the survey + response from the given project and PI for the given allocation + period. If there is no response, return None. + + Parameters + - allocation_period_name (str): the name of an AllocationPeriod + - project_name (str): the name of a Project + - pi_username (str): the username of a PI of the Project + + Returns: + - list of tuples (str, str), if there is a response + - None, if there is no response + + Raises: + - Exception, if any errors occur. + """ + pass + + @abstractmethod + def get_renewal_survey_url(self, allocation_period_name, pi, project_name, + requester): + """Return a unique link to the survey to be filled out by the + given requesting user on behalf of the given project and PI for + the given allocation period. + + Parameters: + - allocation_period_name (str): the name of an AllocationPeriod + - pi (User): the PI of the Project + - project_name (str): the name of a Project + - requester (User): the user making the renewal request and + filling out the survey + + Returns: + - str + + Raises: + - Exception, if any errors occur. + """ + pass diff --git a/coldfront/core/project/utils_/renewal_survey/backends/google_forms.py b/coldfront/core/project/utils_/renewal_survey/backends/google_forms.py new file mode 100644 index 000000000..a8d16c17d --- /dev/null +++ b/coldfront/core/project/utils_/renewal_survey/backends/google_forms.py @@ -0,0 +1,252 @@ +import gspread +import json +import logging +import os + +from django.conf import settings +from django.core.cache import cache + +from coldfront.core.project.utils_.renewal_survey.backends.base import BaseRenewalSurveyBackend + + +logger = logging.getLogger(__name__) + + +class GoogleFormsRenewalSurveyBackend(BaseRenewalSurveyBackend): + """A backend that supports a renewal survey hosted on Google + Forms.""" + + def is_renewal_survey_completed(self, allocation_period_name, project_name, + pi_username): + """Return whether there is a response for the given project and + PI in the Google Sheet for the period.""" + survey_data = self._load_renewal_survey_metadata(allocation_period_name) + + wks = self._get_gspread_wks(survey_data['sheet_id']) + periods_coor = self._gsheet_column_to_index( + survey_data['sheet_data']['allocation_period_col']) + pis_coor = self._gsheet_column_to_index( + survey_data['sheet_data']['pi_username_col']) + projects_coor = self._gsheet_column_to_index( + survey_data['sheet_data']['project_name_col']) + + periods = wks.col_values(periods_coor) + pis = wks.col_values(pis_coor) + projects = wks.col_values(projects_coor) + responses = list(zip(periods, pis, projects)) + + key = (allocation_period_name, pi_username, project_name) + # Search later responses first. + for i in range(len(responses) - 1, 0, -1): + if responses[i] == key: + return True + + return False + + def get_renewal_survey_response(self, allocation_period_name, project_name, + pi_username): + """Fetch the response for the given project and PI in the Google + Sheet for the period. Return None if there is no response.""" + gform_info = self._load_renewal_survey_metadata(allocation_period_name) + if gform_info is None: + return None + + wks = self._get_gspread_wks(gform_info['sheet_id']) + if wks is None: + return None + + pis_column_coor = self._gsheet_column_to_index( + gform_info['sheet_data']['pi_username_col']) + projs_column_coor = self._gsheet_column_to_index( + gform_info['sheet_data']['project_name_col']) + + pis = wks.col_values(pis_column_coor) + projects = wks.col_values(projs_column_coor) + + all_responses = list(zip(pis, projects)) + key = (pi_username, project_name) + + row_ind = None + # Search later responses first. + for i in range(len(all_responses) - 1, 0, -1): + if all_responses[i] == key: + # Correct for Google Sheets not being zero-indexed + row_ind = i + 1 + break + + if row_ind is None: + return None + + questions = wks.row_values(1) + response = wks.row_values(row_ind) + + return zip(questions, response) + + def get_renewal_survey_url(self, allocation_period_name, pi, project_name, + requester): + """Return a pre-filled link to the Google Form for the period, + wherein the following are pre-filled: + - The name of the AllocationPeriod + - The name and username of the PI (User object) + - The name of the project + - The name and username of the requester (User object) + """ + gform_info = self._load_renewal_survey_metadata(allocation_period_name) + if gform_info is None: + return None + + wks = self._get_gspread_wks(gform_info['sheet_id']) + if wks is None: + return None + + BASE_URL_ONE = 'https://docs.google.com/forms/d/e/' + BASE_URL_TWO = '/viewform?usp=pp_url' + + url = BASE_URL_ONE + gform_info['form_id'] + BASE_URL_TWO + + PARAMETER_BASE_ONE = '&entry.' + PARAMETER_BASE_TWO = '=' + + question_ids_dict = gform_info['form_question_ids'] + for question in question_ids_dict.keys(): + value = '' + if question == 'allocation_period': + value = allocation_period_name + elif question == 'pi_name': + value = pi.first_name + '+' + pi.last_name + elif question == 'pi_username': + value = pi.username + elif question == 'project_name': + value = project_name + elif question == 'requester_name': + value = requester.first_name + '+' + \ + requester.last_name + elif question == 'requester_username': + value = requester.username + value = value.replace(' ', '+') + url += PARAMETER_BASE_ONE + question_ids_dict[question] + \ + PARAMETER_BASE_TWO + value + return url + + @staticmethod + def _get_gspread_wks(sheet_id, wks_id=0): + """Given the spreadsheet ID and worksheet ID (default 0) of a + Google Sheet, return a sheet that is editable. + + Raises: + - FileNotFoundError + """ + credentials_file_path = settings.RENEWAL_SURVEY.get( + 'details', {}).get('credentials_file_path', '') + assert isinstance(credentials_file_path, str) + if not os.path.isfile(credentials_file_path): + raise FileNotFoundError( + f'Could not find credentials file: {credentials_file_path}.') + + gc = gspread.service_account(filename=credentials_file_path) + sh = gc.open_by_key(sheet_id) + wks = sh.get_worksheet(wks_id) + + return wks + + @staticmethod + def _gsheet_column_to_index(column_str): + """Convert Google Sheets column (e.g., 'A', 'AA') to index number.""" + index = 0 + for char in column_str: + index = index * 26 + (ord(char.upper()) - ord('A') + 1) + return index + + def _load_renewal_survey_metadata(self, allocation_period_name): + """Return a dict containing metadata about the Google Form and + Google Sheet pertaining to the AllocationPeriod with the given + name. + + The dict should be of the form: + + { + # The ID of the Google Sheet linked to the Google Form. + "sheet_id": "str", + + # The ID of the Google Form. + "form_id": "str", + + # The name of the AllocationPeriod. + "allocation_period": "str", + + # Indices (e.g., "X", "AC") of columns in which specific + # form answers are stored in the sheet. + "sheet_data": { + "allocation_period_col": "str", + "pi_username_col": "str", + "project_name_col": "str", + }, + + # The IDs of form questions, used to pre-fill answers. + "form_question_ids": { + "allocation_period": "str", + "pi_name": "str", + "pi_username": "str", + "project_name": "str", + "requester_name": "str", + "requester_username": "str", + }, + } + + These dicts are stored in a file on local disk, and cached in + Django's caching mechanism. + + Raises: + - FileNotFoundError + - ValueError + """ + renewal_survey_details = settings.RENEWAL_SURVEY.get('details', {}) + cache_key = renewal_survey_details.get('survey_data_cache_key', None) + if cache_key is None: + logger.error( + 'Failed to retrieve cache key from settings.RENEWAL_SURVEY.') + + cache_value = {} + if cache_key is not None and cache_key in cache: + cache_value = cache.get(cache_key) + if allocation_period_name in cache_value: + return cache_value[allocation_period_name] + + metadata = self._load_survey_metadata_from_file(allocation_period_name) + + cache_value[allocation_period_name] = metadata + cache.set(cache_key, cache_value) + + return metadata + + @staticmethod + def _load_survey_metadata_from_file(allocation_period_name): + """Return a dict containing metadata about the Google Form and + Google Sheet pertaining to the AllocationPeriod with the given + name, sourced from a file on local disk. + + Raises: + - FileNotFoundError + - ValueError + """ + renewal_survey_details = settings.RENEWAL_SURVEY.get('details', {}) + metadata_file_path = renewal_survey_details.get( + 'survey_data_file_path', None) + if not os.path.isfile(metadata_file_path): + raise FileNotFoundError( + f'Could not find renewal survey data file: ' + f'{metadata_file_path}.') + + metadata = None + with open(metadata_file_path, 'r') as f: + metadata_dicts = json.load(f) + for metadata_dict in metadata_dicts: + if metadata_dict['allocation_period'] == allocation_period_name: + metadata = metadata_dict + break + + if metadata is None: + raise ValueError( + 'Failed to load survey data for AllocationPeriod from file.') + + return metadata diff --git a/coldfront/core/project/utils_/renewal_survey/backends/permissive.py b/coldfront/core/project/utils_/renewal_survey/backends/permissive.py new file mode 100644 index 000000000..bfb175e72 --- /dev/null +++ b/coldfront/core/project/utils_/renewal_survey/backends/permissive.py @@ -0,0 +1,21 @@ +from coldfront.core.project.utils_.renewal_survey.backends.base import BaseRenewalSurveyBackend + + +class PermissiveRenewalSurveyBackend(BaseRenewalSurveyBackend): + """A backend that always reports that a survey has been + completed.""" + + def is_renewal_survey_completed(self, allocation_period_name, project_name, + pi_username): + """Always report that a survey has been completed.""" + return True + + def get_renewal_survey_response(self, allocation_period_name, project_name, + pi_username): + """Return an empty list of responses.""" + return [] + + def get_renewal_survey_url(self, allocation_period_name, pi, project_name, + requester): + """Return an empty string.""" + return '' diff --git a/coldfront/core/project/utils_/renewal_survey/data/google_forms_survey_data.json.sample b/coldfront/core/project/utils_/renewal_survey/data/google_forms_survey_data.json.sample new file mode 100644 index 000000000..6d0369a64 --- /dev/null +++ b/coldfront/core/project/utils_/renewal_survey/data/google_forms_survey_data.json.sample @@ -0,0 +1,20 @@ +[ + { + "sheet_id": "", + "form_id": "", + "allocation_period": "", + "sheet_data": { + "allocation_period_col": "", + "pi_username_col": "", + "project_name_col": "" + }, + "form_question_ids": { + "allocation_period": "", + "pi_name": "", + "pi_username": "", + "project_name": "", + "requester_name": "", + "requester_username": "" + } + } +] \ No newline at end of file diff --git a/coldfront/core/project/utils_/renewal_utils.py b/coldfront/core/project/utils_/renewal_utils.py index e8ffded20..2a4a044ef 100644 --- a/coldfront/core/project/utils_/renewal_utils.py +++ b/coldfront/core/project/utils_/renewal_utils.py @@ -6,6 +6,7 @@ from coldfront.core.allocation.models import AllocationRenewalRequestStatusChoice from coldfront.core.allocation.models import AllocationStatusChoice from coldfront.core.allocation.utils import get_project_compute_allocation +from coldfront.core.allocation.utils import prorated_allocation_amount from coldfront.core.project.models import Project from coldfront.core.project.models import ProjectAllocationRequestStatusChoice from coldfront.core.project.models import ProjectStatusChoice @@ -579,6 +580,33 @@ def allocation_renewal_request_state_status(request): name='Under Review') +def set_allocation_renewal_request_eligibility(request, status, justification, + timestamp=None): + """Update the given AllocationRenewalRequest to note whether the PI + of the request is eligible for renewal, with the following: + - A str 'status' denoting eligibility (one of 'Pending', + 'Approved', 'Denied'), + - A str 'justification' with admin comments, and + - An optional str ISO 8601 'timestamp'. If one is not given, the + current time is used. + + Based on 'status', also update the status of the request (e.g., if + the PI is ineligible, the request's status should have status + 'Denied'. + """ + if timestamp is not None: + assert isinstance(timestamp, str) + else: + timestamp = utc_now_offset_aware().isoformat() + request.state['eligibility'] = { + 'status': status, + 'justification': justification, + 'timestamp': timestamp, + } + request.status = allocation_renewal_request_state_status(request) + request.save() + + class AllocationRenewalRunnerBase(object): """A base class that Runners for handling AllocationRenewalsRequests should inherit from.""" diff --git a/coldfront/core/project/views_/new_project_views/approval_views.py b/coldfront/core/project/views_/new_project_views/approval_views.py index d4dcb0899..2332e11e7 100644 --- a/coldfront/core/project/views_/new_project_views/approval_views.py +++ b/coldfront/core/project/views_/new_project_views/approval_views.py @@ -1,11 +1,10 @@ from coldfront.core.allocation.models import AllocationRenewalRequest from coldfront.core.allocation.utils import annotate_queryset_with_allocation_period_not_started_bool -from coldfront.core.allocation.utils import prorated_allocation_amount +from coldfront.core.allocation.utils import calculate_service_units_to_allocate from coldfront.core.project.forms import MemorandumSignedForm from coldfront.core.project.forms import ReviewDenyForm from coldfront.core.project.forms import ReviewStatusForm from coldfront.core.project.forms_.new_project_forms.request_forms import NewProjectExtraFieldsFormFactory -from coldfront.core.project.forms_.new_project_forms.request_forms import SavioProjectExtraFieldsForm from coldfront.core.project.forms_.new_project_forms.request_forms import SavioProjectSurveyForm from coldfront.core.project.forms_.new_project_forms.approval_forms import SavioProjectReviewSetupForm from coldfront.core.project.forms_.new_project_forms.approval_forms import VectorProjectReviewSetupForm @@ -143,21 +142,13 @@ def get_service_units_to_allocate(self): 'num_service_units'] num_service_units = Decimal(f'{num_service_units_int:.2f}') else: - allowance_name = self.request_obj.computing_allowance.name - - allocation_period = self.request_obj.allocation_period kwargs = {} + allocation_period = self.request_obj.allocation_period if allocation_period: - kwargs['is_timed'] = True kwargs['allocation_period'] = allocation_period - num_service_units = Decimal( - self.interface.service_units_from_name( - allowance_name, **kwargs)) - - if self.computing_allowance_obj.are_service_units_prorated(): - num_service_units = prorated_allocation_amount( - num_service_units, self.request_obj.request_time, - self.request_obj.allocation_period) + num_service_units = calculate_service_units_to_allocate( + self.computing_allowance_obj, self.request_obj.request_time, + **kwargs) return num_service_units def get_survey_form(self): diff --git a/coldfront/core/project/views_/new_project_views/request_views.py b/coldfront/core/project/views_/new_project_views/request_views.py index e4bfa52d8..2a8d41d12 100644 --- a/coldfront/core/project/views_/new_project_views/request_views.py +++ b/coldfront/core/project/views_/new_project_views/request_views.py @@ -314,6 +314,9 @@ def done(self, form_list, **kwargs): @staticmethod def condition_dict(): + """Return a mapping from a string index `i` into FORMS + (zero-indexed) to a function determining whether FORMS[int(i)] + should be included.""" view = SavioProjectRequestWizard return { '1': view.show_allocation_period_form_condition, diff --git a/coldfront/core/project/views_/renewal_views/approval_views.py b/coldfront/core/project/views_/renewal_views/approval_views.py index 796e3af8c..0652f65bc 100644 --- a/coldfront/core/project/views_/renewal_views/approval_views.py +++ b/coldfront/core/project/views_/renewal_views/approval_views.py @@ -1,26 +1,24 @@ from coldfront.core.allocation.models import AllocationRenewalRequest from coldfront.core.allocation.utils import annotate_queryset_with_allocation_period_not_started_bool -from coldfront.core.allocation.utils import prorated_allocation_amount +from coldfront.core.allocation.utils import calculate_service_units_to_allocate from coldfront.core.project.forms import ReviewDenyForm from coldfront.core.project.forms import ReviewStatusForm from coldfront.core.project.models import ProjectAllocationRequestStatusChoice -from coldfront.core.project.forms_.renewal_forms.request_forms import ProjectRenewalSurveyForm from coldfront.core.project.utils_.renewal_utils import AllocationRenewalApprovalRunner from coldfront.core.project.utils_.renewal_utils import AllocationRenewalDenialRunner from coldfront.core.project.utils_.renewal_utils import AllocationRenewalProcessingRunner from coldfront.core.project.utils_.renewal_utils import allocation_renewal_request_denial_reason from coldfront.core.project.utils_.renewal_utils import allocation_renewal_request_latest_update_timestamp from coldfront.core.project.utils_.renewal_utils import allocation_renewal_request_state_status +from coldfront.core.project.utils_.renewal_survey import get_renewal_survey_response +from coldfront.core.project.utils_.renewal_utils import set_allocation_renewal_request_eligibility from coldfront.core.resource.utils_.allowance_utils.computing_allowance import ComputingAllowance -from coldfront.core.resource.utils_.allowance_utils.interface import ComputingAllowanceInterface from coldfront.core.utils.common import display_time_zone_current_date from coldfront.core.utils.common import format_date_month_name_day_year from coldfront.core.utils.common import utc_now_offset_aware from coldfront.core.utils.email.email_strategy import DropEmailStrategy from coldfront.core.utils.email.email_strategy import EnqueueEmailStrategy -from decimal import Decimal - from django.conf import settings from django.contrib import messages from django.contrib.auth.mixins import LoginRequiredMixin @@ -35,6 +33,8 @@ from django.views.generic import TemplateView from django.views.generic.edit import FormView +from flags.state import flag_enabled + import iso8601 import logging @@ -99,8 +99,13 @@ def get_context_data(self, **kwargs): logger.exception(e) messages.error(self.request, self.error_message) context['allocation_amount'] = 'Failed to compute.' + if flag_enabled('RENEWAL_SURVEY_ENABLED'): + context['survey_response'] = get_renewal_survey_response( + self.request_obj.allocation_period.name, + self.request_obj.post_project.name, + self.request_obj.pi.username) context['has_survey_answers'] = bool( - self.request_obj.renewal_survey_answers) + context.get('survey_response', None)) return context @staticmethod @@ -109,17 +114,11 @@ def get_redirect_url(pk): 'pi-allocation-renewal-request-detail', kwargs={'pk': pk}) def get_service_units_to_allocate(self): - """Return the number of service units to allocate to the project - if it were to be approved now.""" - num_service_units = Decimal( - ComputingAllowanceInterface().service_units_from_name( - self.computing_allowance_obj.get_name(), - is_timed=True, allocation_period=self.allocation_period_obj)) - if self.computing_allowance_obj.are_service_units_prorated(): - num_service_units = prorated_allocation_amount( - num_service_units, self.request_obj.request_time, - self.allocation_period_obj) - return num_service_units + """Return the number of service units to allocate to the + project.""" + return calculate_service_units_to_allocate( + self.computing_allowance_obj, self.request_obj.request_time, + allocation_period=self.allocation_period_obj) def set_common_context_data(self, context): """Given a dictionary of context variables to include in the @@ -127,11 +126,6 @@ def set_common_context_data(self, context): context['renewal_request'] = self.request_obj context['computing_allowance_name'] = \ self.computing_allowance_obj.get_name() - context['survey_form'] = ProjectRenewalSurveyForm( - initial=self.request_obj.renewal_survey_answers, - disable_fields=True) - context['has_survey_answers'] = bool( - self.request_obj.renewal_survey_answers) def set_objs(self, pk): self.request_obj = get_object_or_404( @@ -375,15 +369,9 @@ def form_valid(self, form): form_data = form.cleaned_data status = form_data['status'] justification = form_data['justification'] - timestamp = utc_now_offset_aware().isoformat() - self.request_obj.state['eligibility'] = { - 'status': status, - 'justification': justification, - 'timestamp': timestamp, - } - self.request_obj.status = allocation_renewal_request_state_status( - self.request_obj) - self.request_obj.save() + + set_allocation_renewal_request_eligibility( + self.request_obj, status, justification) if status == 'Denied': runner = AllocationRenewalDenialRunner(self.request_obj) diff --git a/coldfront/core/project/views_/renewal_views/request_views.py b/coldfront/core/project/views_/renewal_views/request_views.py index ba09438a7..a8a901c8b 100644 --- a/coldfront/core/project/views_/renewal_views/request_views.py +++ b/coldfront/core/project/views_/renewal_views/request_views.py @@ -1,5 +1,4 @@ from coldfront.core.allocation.models import Allocation -from coldfront.core.allocation.models import AllocationPeriod from coldfront.core.allocation.models import AllocationRenewalRequest from coldfront.core.allocation.models import AllocationRenewalRequestStatusChoice from coldfront.core.allocation.models import AllocationStatusChoice @@ -25,6 +24,7 @@ from coldfront.core.project.utils_.renewal_utils import send_new_allocation_renewal_request_admin_notification_email from coldfront.core.project.utils_.renewal_utils import send_new_allocation_renewal_request_pi_notification_email from coldfront.core.project.utils_.renewal_utils import send_new_allocation_renewal_request_pooling_notification_email +from coldfront.core.project.utils_.renewal_survey import get_renewal_survey_url from coldfront.core.resource.models import Resource from coldfront.core.resource.utils import get_primary_compute_resource from coldfront.core.resource.utils_.allowance_utils.computing_allowance import ComputingAllowance @@ -131,7 +131,7 @@ def __init__(self, *args, **kwargs): @staticmethod def create_allocation_renewal_request(requester, pi, computing_allowance, allocation_period, pre_project, - post_project, renewal_survey_answers, + post_project, new_project_request=None): """Create a new AllocationRenewalRequest.""" request_kwargs = dict() @@ -142,7 +142,6 @@ def create_allocation_renewal_request(requester, pi, computing_allowance, request_kwargs['status'] = \ AllocationRenewalRequestStatusChoice.objects.get( name='Under Review') - request_kwargs['renewal_survey_answers'] = renewal_survey_answers request_kwargs['pre_project'] = pre_project request_kwargs['post_project'] = post_project request_kwargs['new_project_request'] = new_project_request @@ -270,6 +269,14 @@ def get_context_data(self, form, **kwargs): context = super().get_context_data(form=form, **kwargs) current_step = int(self.steps.current) self.__set_data_from_previous_steps(current_step, context) + + if current_step == self.step_numbers_by_form_name['renewal_survey']: + context['renewal_survey_url'] = get_renewal_survey_url( + context['allocation_period'].name, + context['PI'].user, + context['requested_project'].name, + self.request.user) + return context def get_form_kwargs(self, step=None): @@ -318,6 +325,13 @@ def get_form_kwargs(self, step=None): kwargs['computing_allowance'] = self.computing_allowance elif step == self.step_numbers_by_form_name['new_project_survey']: kwargs['computing_allowance'] = self.computing_allowance + elif step == self.step_numbers_by_form_name['renewal_survey']: + tmp = {} + self.__set_data_from_previous_steps(step, tmp) + kwargs['project_name'] = tmp['requested_project'].name + kwargs['allocation_period_name'] = tmp['allocation_period'].name + kwargs['pi_username'] = tmp['PI'].user.username + return kwargs def get_template_names(self): @@ -360,7 +374,6 @@ def done(self, form_list, **kwargs): request = self.create_allocation_renewal_request( self.request.user, pi, self.computing_allowance, allocation_period, tmp['current_project'], requested_project, - tmp['renewal_survey_answers'], new_project_request=new_project_request) self.send_emails(request) @@ -374,6 +387,9 @@ def done(self, form_list, **kwargs): @staticmethod def condition_dict(): + """Return a mapping from a string index `i` into FORMS + (zero-indexed) to a function determining whether FORMS[int(i)] + should be included.""" view = AllocationRenewalRequestView return { '3': view.show_project_selection_form_condition, @@ -408,19 +424,8 @@ def show_project_selection_form_condition(self): return cleaned_data.get('preference', None) in preferences def show_renewal_survey_form_condition(self): - """Only show the renewal survey form for a particular period. - - TODO: This period has been hard-coded for the short-term. A - longer-term solution without hard-coding must be applied prior - to the start of the period following it. - """ - step_name = 'allocation_period' - step = str(self.step_numbers_by_form_name[step_name]) - cleaned_data = self.get_cleaned_data_for_step(step) or {} - allocation_period = cleaned_data.get('allocation_period', None) - expected_allocation_period = AllocationPeriod.objects.get( - name='Allowance Year 2024 - 2025') - return allocation_period == expected_allocation_period + """Only show the renewal survey form if a survey is required.""" + return flag_enabled('RENEWAL_SURVEY_ENABLED') def __get_survey_data(self, form_data): """Return provided survey data.""" @@ -571,12 +576,6 @@ def __set_data_from_previous_steps(self, step, dictionary): dictionary.update(data) dictionary['requested_project'] = data['name'] - renewal_survey_form_step = self.step_numbers_by_form_name[ - 'renewal_survey'] - if step > renewal_survey_form_step: - data = self.get_cleaned_data_for_step(str(renewal_survey_form_step)) - dictionary['renewal_survey_answers'] = data or {} - class AllocationRenewalRequestUnderProjectView(LoginRequiredMixin, UserPassesTestMixin, @@ -612,13 +611,6 @@ def __init__(self, *args, **kwargs): self.step_numbers_by_form_name = { name: i for i, (name, _) in enumerate(self.FORMS)} - @staticmethod - def condition_dict(): - view = AllocationRenewalRequestUnderProjectView - return { - '2': view.show_renewal_survey_form_condition, - } - def dispatch(self, request, *args, **kwargs): pk = self.kwargs.get('pk') self.project_obj = get_object_or_404(Project, pk=pk) @@ -647,8 +639,7 @@ def done(self, form_list, **kwargs): request = self.create_allocation_renewal_request( self.request.user, pi, self.computing_allowance, - allocation_period, self.project_obj, self.project_obj, - tmp['renewal_survey_answers']) + allocation_period, self.project_obj, self.project_obj) self.send_emails(request) except Exception as e: @@ -663,6 +654,14 @@ def get_context_data(self, form, **kwargs): context = super().get_context_data(form=form, **kwargs) current_step = int(self.steps.current) self.__set_data_from_previous_steps(current_step, context) + + if current_step == self.step_numbers_by_form_name['renewal_survey']: + context['renewal_survey_url'] = get_renewal_survey_url( + context['allocation_period'].name, + context['PI'].user, + context['requested_project'].name, + self.request.user) + return context def get_form_kwargs(self, step=None): @@ -677,26 +676,17 @@ def get_form_kwargs(self, step=None): kwargs['allocation_period_pk'] = getattr( tmp.get('allocation_period', None), 'pk', None) kwargs['project_pks'] = [self.project_obj.pk] + elif step == self.step_numbers_by_form_name['renewal_survey']: + tmp = {} + self.__set_data_from_previous_steps(step, tmp) + kwargs['project_name'] = tmp['requested_project'].name + kwargs['allocation_period_name'] = tmp['allocation_period'].name + kwargs['pi_username'] = tmp['PI'].user.username return kwargs def get_template_names(self): return [self.TEMPLATES[self.FORMS[int(self.steps.current)][0]]] - def show_renewal_survey_form_condition(self): - """Only show the renewal survey form for a particular period. - - TODO: This period has been hard-coded for the short-term. A - longer-term solution without hard-coding must be applied prior - to the start of the period following it. - """ - step_name = 'allocation_period' - step = str(self.step_numbers_by_form_name[step_name]) - cleaned_data = self.get_cleaned_data_for_step(step) or {} - allocation_period = cleaned_data.get('allocation_period', None) - expected_allocation_period = AllocationPeriod.objects.get( - name='Allowance Year 2024 - 2025') - return allocation_period == expected_allocation_period - def test_func(self): """Allow superusers and users who are active Managers or Principal Investigators on the Project and who have signed the @@ -719,6 +709,20 @@ def test_func(self): 'Project.') messages.error(self.request, message) + @staticmethod + def condition_dict(): + """Return a mapping from a string index `i` into FORMS + (zero-indexed) to a function determining whether FORMS[int(i)] + should be included.""" + view = AllocationRenewalRequestUnderProjectView + return { + '2': view.show_renewal_survey_form_condition, + } + + def show_renewal_survey_form_condition(self): + """Only show the renewal survey form if a survey is required.""" + return flag_enabled('RENEWAL_SURVEY_ENABLED') + def __set_data_from_previous_steps(self, step, dictionary): """Update the given dictionary with data from previous steps.""" allocation_period_form_step = self.step_numbers_by_form_name[ @@ -748,9 +752,3 @@ def __set_data_from_previous_steps(self, step, dictionary): dictionary['breadcrumb_pooling_preference'] = \ form_class.SHORT_DESCRIPTIONS.get( pooling_preference, 'Unknown') - - renewal_survey_form_step = self.step_numbers_by_form_name[ - 'renewal_survey'] - if step > renewal_survey_form_step: - data = self.get_cleaned_data_for_step(str(renewal_survey_form_step)) - dictionary['renewal_survey_answers'] = data or {} diff --git a/coldfront/core/resource/utils_/allowance_utils/computing_allowance.py b/coldfront/core/resource/utils_/allowance_utils/computing_allowance.py index 08868dc83..a152e2160 100644 --- a/coldfront/core/resource/utils_/allowance_utils/computing_allowance.py +++ b/coldfront/core/resource/utils_/allowance_utils/computing_allowance.py @@ -1,3 +1,5 @@ +from django.db.models import Q + from flags.state import flag_enabled from coldfront.core.resource.models import Resource @@ -107,7 +109,7 @@ def is_renewable(self): return self.is_periodic() def is_renewal_supported(self): - """Return whether there is support for renewing the + """Return whether there is UI support for renewing the allowance.""" allowance_names = [] if flag_enabled('BRC_ONLY'): @@ -131,6 +133,27 @@ def get_name(self): """Return the name of the underlying Resource.""" return self._name + def get_period_filters(self): + """Return a Django Q object that can be used to filter for + AllocationPeriods associated with the allowance. + + If none are applicable, return None. + """ + yearly_q = Q(name__startswith='Allowance Year') + if flag_enabled('BRC_ONLY'): + if self.is_yearly(): + return yearly_q + elif self.is_instructional(): + instructional_q = ( + Q(name__startswith='Fall Semester') | + Q(name__startswith='Spring Semester') | + Q(name__startswith='Summer Sessions')) + return instructional_q + if flag_enabled('LRC_ONLY'): + if self.is_yearly(): + return yearly_q + return None + def get_resource(self): """Return the underlying Resource.""" return self._resource diff --git a/coldfront/core/utils/management/commands/export_data.py b/coldfront/core/utils/management/commands/export_data.py index b1a216885..ff50e171b 100644 --- a/coldfront/core/utils/management/commands/export_data.py +++ b/coldfront/core/utils/management/commands/export_data.py @@ -13,7 +13,7 @@ from coldfront.core.statistics.models import Job from coldfront.core.project.models import Project, ProjectStatusChoice, \ SavioProjectAllocationRequest, VectorProjectAllocationRequest -from coldfront.core.project.forms_.renewal_forms.request_forms import ProjectRenewalSurveyForm +from coldfront.core.project.forms_.renewal_forms.request_forms import DeprecatedProjectRenewalSurveyForm from coldfront.core.resource.utils_.allowance_utils.interface import ComputingAllowanceInterface from coldfront.core.utils.common import display_time_zone_date_to_utc_datetime @@ -520,14 +520,16 @@ def _swap_form_answer_id_for_text(_survey, _multiple_choice_fields): allocation_period__name=allocation_period) _surveys = list(allocation_requests.values_list('renewal_survey_answers', flat=True)) + _surveys = [i for i in _surveys if i != {}] if len(_surveys) == 0: raise Exception("There are no valid renewal requests in the specified allocation period.") + if {} in _surveys: raise Exception("This allocation period does not have an associated survey.") surveys = [] # Create dict of multiple choice fields to replace field IDs with text. ID : text multiple_choice_fields = {} - form = ProjectRenewalSurveyForm() + form = DeprecatedProjectRenewalSurveyForm() for k, v in form.fields.items(): # Only ChoiceField or MultipleChoiceField (in this specific survey form) have choices if (isinstance(v, forms.MultipleChoiceField)) or (isinstance(v, forms.ChoiceField)): diff --git a/coldfront/core/utils/tests/test_export_data.py b/coldfront/core/utils/tests/test_export_data.py index 8df3cd057..237774cb9 100644 --- a/coldfront/core/utils/tests/test_export_data.py +++ b/coldfront/core/utils/tests/test_export_data.py @@ -27,7 +27,7 @@ ProjectUser, ProjectUserStatusChoice, ProjectUserRoleChoice, \ ProjectAllocationRequestStatusChoice, SavioProjectAllocationRequest, \ VectorProjectAllocationRequest -from coldfront.core.project.forms_.renewal_forms.request_forms import ProjectRenewalSurveyForm +from coldfront.core.project.forms_.renewal_forms.request_forms import DeprecatedProjectRenewalSurveyForm from coldfront.core.project.utils_.renewal_utils import get_current_allowance_year_period from coldfront.core.resource.models import Resource from coldfront.core.resource.utils_.allowance_utils.constants import BRCAllowances @@ -1167,7 +1167,7 @@ def test_get_renewal_survey_responses_allowance_type(self): def swap_form_answer_id_for_text(survey): ''' Takes a survey, a dict mapping survey question IDs to answer IDs. - Uses ProjectRenewalSurveyForm. + Uses DeprecatedProjectRenewalSurveyForm. Swaps answer IDs for answer text, then question IDs for question text. Returns the modified survey. @@ -1176,7 +1176,7 @@ def swap_form_answer_id_for_text(survey): survey : survey to modify ''' multiple_choice_fields = {} - form = ProjectRenewalSurveyForm() + form = DeprecatedProjectRenewalSurveyForm() for k, v in form.fields.items(): # Only ChoiceField or MultipleChoiceField # (in this specific survey form) have choices diff --git a/requirements.txt b/requirements.txt index 96a0a99b9..038956950 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,6 +18,7 @@ django-model-utils==4.1.1 django-phonenumber-field==5.1.0 django-picklefield==2.0 django-q==1.0.1 +django-redis==5.4.0 django-sesame==3.1 django-settings-export==1.2.1 django-simple-history==3.0.0