Merge pull request #489 from ucb-rit/develop

Update Ansible configuration; add testing and debugging tools; fix bugs

.github/workflows/django_testing_ci.yml

@@ -0,0 +1,89 @@
+# Github Actions workflow that runs the test suite on the master branch
+# every night, as well as tests new pull requests.
+
+name: Django Testing CI
+
+on:
+  schedule: # Run every night at 3 AM
+    - cron: '0 10 * * *' # 10 AM UTC = 3 AM PST
+  push: # Run when pushes are made on the master and develop branch
+    branches: [ "master", "develop" ]
+  pull_request: # Runs when pull request to develop or master is opened,
+                # reopened, or updated with a new commit.
+    branches: [ "master", "develop" ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest # Default github actions environment for Linux
+
+    services:
+      postgres: # Set up a database container with the following specifications
+        image: postgres:9.6.24-alpine3.15
+        env:
+          POSTGRES_DB: cf_brc_db
+          POSTGRES_PASSWORD: test
+          POSTGRES_PORT: 5432
+          POSTGRES_USER: test
+        ports:
+        - 5432:5432
+        options: >- # Actions run continues when database health is asserted
+          --health-cmd pg_isready
+          --health-interval 2s
+          --health-timeout 3s
+          --health-retries 15
+
+    steps: # Steps to run to set up testing
+    - name: Checkout the current commit
+      uses: actions/checkout@v3
+
+    - name: Set up Python 3.6.8
+      uses: actions/setup-python@v3
+      with:
+        python-version: 3.6.8
+
+    - name: Cache and/or Install apache2-dev needed for testing suite
+      uses: awalsh128/cache-apt-pkgs-action@latest
+      with:
+          packages: apache2-dev
+
+    - name: Cache Python packages # Use a cached installation of Python packages
+      id: cache-python
+      uses: actions/cache@v3
+      with:
+        path: ~/venv
+        key: ${{ runner.os }}-python-packages-${{ hashFiles('requirements.txt') }}
+
+    - if: ${{ steps.cache-python.outputs.cache-hit != 'true' }} # If a cache is not found
+      name: Install Python packages
+      run: |
+        python3.6 -m venv ~/venv
+        source ~/venv/bin/activate
+        pip install -r requirements.txt
+
+    - name: Create settings files from samples and create log files needed for testing
+      run: |
+        # Create log files
+        sudo mkdir -p /var/log/user_portals/cf_mybrc
+        sudo touch /var/log/user_portals/cf_mybrc/cf_mybrc_portal.log
+        sudo touch /var/log/user_portals/cf_mybrc/cf_mybrc_api.log
+        
+        # Modify log file permssions to allow testing to function properly
+        sudo chmod 775 /var/log/user_portals/cf_mybrc
+        sudo chmod 666 /var/log/user_portals/cf_mybrc/cf_mybrc_portal.log
+        sudo chmod 666 /var/log/user_portals/cf_mybrc/cf_mybrc_api.log
+        
+        # Give Apache permission to logs
+        sudo chown -R :www-data /var/log/user_portals/cf_mybrc
+        
+        # Get setting configuration from samples
+        cp coldfront/config/local_strings.py.sample coldfront/config/local_strings.py
+        cp coldfront/config/local_settings.py.sample coldfront/config/local_settings.py
+        cp coldfront/config/test_settings.py.sample coldfront/config/test_settings.py
+
+    - name: Run Tests
+      run: |
+        source ~/venv/bin/activate
+        python manage.py migrate
+        python manage.py test
+

README.md

@@ -53,6 +53,7 @@ that quotes need not be provided, except in the list variable.
    redis_passwd: password_here
    from_email: [email protected]
    admin_email: [email protected]
+   email_admin_list: ["[email protected]"]
    request_approval_cc_list: ["[email protected]"]
    ```
 8. Provision the VM. This should run the Ansible playbook. Expect this to take

bootstrap/ansible/main.copyme

@@ -1,3 +1,11 @@
+###############################################################################
+# Ansible Settings
+###############################################################################
+
+# Types of Ansible tasks to run by default.
+provisioning_tasks: true
+common_tasks: true
+
 ###############################################################################
 # General Settings
 ###############################################################################
@@ -36,6 +44,37 @@ wsgi_conf_file_name: cf_mybrc_wsgi.conf
 # TODO: For LRC, use the substring 'cf_lrc'.
 wsgi_conf_log_prefix: cf_brc
 
+# LRC Cloudflare settings.
+# Whether the web server is behind Cloudflare.
+# TODO: For the LRC production deployment, enable Cloudflare, since LBL
+# TODO: requires that web servers visible to the Internet be placed behind it.
+# TODO: https://commons.lbl.gov/display/cpp/Open+Web+Server+Requirements
+cloudflare_enabled: false
+# A list of Cloudflare's IP ranges.
+# TODO: Keep it up-to-date with: https://www.cloudflare.com/ips/.
+cloudflare_ip_ranges: [
+    103.21.244.0/22,
+    103.22.200.0/22,
+    103.31.4.0/22,
+    104.16.0.0/13,
+    104.24.0.0/14,
+    108.162.192.0/18,
+    131.0.72.0/22,
+    141.101.64.0/18,
+    162.158.0.0/15,
+    172.64.0.0/13,
+    173.245.48.0/20,
+    188.114.96.0/20,
+    190.93.240.0/20,
+    197.234.240.0/22,
+    198.41.128.0/17
+]
+# The name of the server, which should differ from the name of the website.
+# Source: See Open Web Server Requirements link above.
+# TODO: Set this to e.g., mylrc-local.lbl.gov for mylrc.lbl.gov if Cloudflare
+# TODO: is enabled.
+cloudflare_local_server_name:
+
 # CILogon client settings.
 # TODO: Set these, needed only if SSO should be enabled.
 cilogon_app_client_id: ""
@@ -60,7 +99,7 @@ portal_name: "MyBRC"
 program_name_long: "Berkeley Research Computing"
 program_name_short: "BRC"
 primary_cluster_name: "Savio"
-# TODO: For MyLRC, use "https://it.lbl.gov/resource/hpc/for-users/".
+# TODO: For MyLRC, use "https://it.lbl.gov/service/scienceit/high-performance-computing/lrc/".
 center_user_guide: "https://docs-research-it.berkeley.edu/services/high-performance-computing/user-guide/"
 # TODO: For MyLRC, use "https://it.lbl.gov/resource/hpc/for-users/getting-started/".
 center_login_guide: "https://docs-research-it.berkeley.edu/services/high-performance-computing/user-guide/logging-brc-clusters/#Logging-in"
@@ -90,10 +129,6 @@ allow_all_jobs: false
 # The URL of the Sentry instance to send errors to.
 sentry_dsn: ""
 
-# Types of Ansible tasks to run by default.
-provisioning_tasks: True
-common_tasks: True
-
 ###############################################################################
 # staging_settings
 ###############################################################################
@@ -120,10 +155,15 @@ common_tasks: True
 # ssl_enabled: false
 # ssl_certificate_file: /etc/ssl/ssl_certificate.file
 # ssl_certificate_key_file: /etc/ssl/ssl_certificate_key.file
+# # An optional chain file.
 # ssl_certificate_chain_file: /etc/ssl/ssl_certification_chain.file
 
-# # An IP range, in CIDR notation, to which the REST API is accessible.
-# ip_range_with_api_access: 0.0.0.0/24
+# # Zero or more space-separated IP ranges, in CIDR notation, to which the REST
+# # API is accessible. If none are given, API access is not restricted.
+# ip_range_with_api_access:
+
+# # IP addresses other than 127.0.0.1 that can view the django debug toolbar.
+# debug_toolbar_ips: []
 
 # # Email settings.
 # email_port: 25
@@ -133,8 +173,12 @@ common_tasks: True
 # # TODO: For LRC, use the substring 'MyLRC'.
 # email_subject_prefix: '[MyBRC-User-Portal]'
 
+# # A list of admin email addresses to be notified about new requests and other
+# # events.
+# # TODO: Set these addresses to yours.
+# email_admin_list: []
 # # A list of email addresses to CC when certain requests are processed.
-# # TODO: Set this address to yours.
+# # TODO: Set these addresses to yours.
 # request_approval_cc_list: []
 
 ###############################################################################
@@ -166,12 +210,16 @@ common_tasks: True
 # ssl_enabled: true
 # ssl_certificate_file: /etc/ssl/ssl_certificate.file
 # ssl_certificate_key_file: /etc/ssl/ssl_certificate_key.file
+# # An optional chain file.
 # ssl_certificate_chain_file: /etc/ssl/ssl_certification_chain.file
 
-# # One or more space-separated IP ranges, in CIDR notation, to which the REST
-# # API is accessible.
+# # Zero or more space-separated IP ranges, in CIDR notation, to which the REST
+# # API is accessible. If none are given, API access is not restricted.
 # ip_range_with_api_access: 10.0.0.0/8
 
+# # IP addresses other than 127.0.0.1 that can view the django debug toolbar.
+# debug_toolbar_ips: []
+
 # # Email settings.
 # email_port: 25
 # # TODO: Set these addresses to yours.
@@ -180,9 +228,13 @@ common_tasks: True
 # # TODO: For LRC, use the substring 'MyLRC'.
 # email_subject_prefix: '[MyBRC-User-Portal]'
 
+# # A list of admin email addresses to be notified about new requests and other
+# # events.
+# # TODO: Set these addresses to yours.
+# email_admin_list: []
 # # A list of email addresses to CC when certain requests are processed.
-# # TODO: Set this address to yours.
-# request_approval_cc_list: ['[email protected]']
+# # TODO: Set these addresses to yours.
+# request_approval_cc_list: []
 
 ###############################################################################
 # dev_settings
@@ -210,10 +262,15 @@ common_tasks: True
 # ssl_enabled: false
 # ssl_certificate_file: /etc/ssl/ssl_certificate.file
 # ssl_certificate_key_file: /etc/ssl/ssl_certificate_key.file
+# # An optional chain file.
 # ssl_certificate_chain_file: /etc/ssl/ssl_certification_chain.file
 
-# # An IP range, in CIDR notation, to which the REST API is accessible.
-# ip_range_with_api_access: 0.0.0.0/0
+# # Zero or more space-separated IP ranges, in CIDR notation, to which the REST
+# # API is accessible. If none are given, API access is not restricted.
+# ip_range_with_api_access:
+
+# # IP addresses other than 127.0.0.1 that can view the django debug toolbar.
+# debug_toolbar_ips: ['10.0.2.2']  # 10.0.2.2 is the vagrant host.
 
 # # Email settings.
 # email_port: 1025
@@ -223,6 +280,10 @@ common_tasks: True
 # # TODO: For LRC, use the substring 'MyLRC'.
 # email_subject_prefix: '[MyBRC-User-Portal]'
 
+# # A list of admin email addresses to be notified about new requests and other
+# # events.
+# # TODO: Set these addresses to yours.
+# email_admin_list: ['[email protected]']
 # # A list of email addresses to CC when certain requests are processed.
-# # TODO: Set this address to yours.
+# # TODO: Set these addresses to yours.
 # request_approval_cc_list: ['[email protected]']

bootstrap/ansible/playbook.yml

@@ -302,15 +302,76 @@
                 state: started
                 enabled: yes
 
-            - name: Permit http and https traffic in public zone
+            - name: Permit http traffic in public zone
               ansible.posix.firewalld:
                 zone: public
-                service: "{{ item }}"
+                service: http
                 state: enabled
                 permanent: yes
-              loop:
-                - http
-                - https
+
+            - name: Run Cloudflare Tasks
+              block:
+                # 443/tcp should only be allowed in Cloudflare IP ranges.
+                # https://commons.lbl.gov/pages/viewpage.action?pageId=203489943
+                - name: Do not permit https traffic in public zone
+                  ansible.posix.firewalld:
+                    zone: public
+                    service: https
+                    state: disabled
+                    permanent: yes
+
+                - name: Deny 443/tcp in public zone
+                  ansible.posix.firewalld:
+                    zone: public
+                    port: 443/tcp
+                    state: disabled
+                    permanent: yes
+
+                - name: Create firewalld zone for Cloudflare IP ranges
+                  ansible.posix.firewalld:
+                    zone: cloudflare
+                    state: present
+                    permanent: yes
+
+                # Firewalld must be reloaded after zone transactions.
+                # https://docs.ansible.com/ansible/latest/collections/ansible/posix/firewalld_module.html#notes
+                - name: Reload firewalld service
+                  service:
+                    name: firewalld
+                    state: reloaded
+
+                - name: Add Cloudflare IP ranges to Cloudflare zone
+                  ansible.posix.firewalld:
+                    zone: cloudflare
+                    source: "{{ item }}"
+                    permanent: yes
+                    state: enabled
+                  loop: "{{ cloudflare_ip_ranges }}"
+
+                - name: Permit http and https traffic in Cloudflare zone
+                  ansible.posix.firewalld:
+                    zone: cloudflare
+                    service: "{{ item }}"
+                    state: enabled
+                    permanent: yes
+                  loop:
+                    - http
+                    - https
+
+                # Log the original client IP address of each request instead of the Cloudflare IP.
+                # Source: https://support.cloudflare.com/hc/en-us/articles/200170786-Restoring-original-visitor-IPs#C5XWe97z77b3XZV
+                - name: Update httpd combined LogFormat in accordance with mod_remoteip
+                  lineinfile:
+                    path: /etc/httpd/conf/httpd.conf
+                    regexp: '^    LogFormat .+ combined$'
+                    line: '    LogFormat "%a %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined'
+
+              when: cloudflare_enabled
+
+            - name: Reload firewalld service
+              service:
+                name: firewalld
+                state: reloaded
 
           when: flag_lrc_enabled
 

bootstrap/ansible/settings_template.tmpl

@@ -21,7 +21,9 @@ CENTER_PROJECT_RENEWAL_HELP_URL = CENTER_BASE_URL + '/help'
 
 EMAIL_PORT = {{ email_port }}
 EMAIL_SUBJECT_PREFIX = '{{ email_subject_prefix }}'
-EMAIL_ADMIN_LIST = ['admin@{{ hostname }}']
+# A list of admin email addresses to be notified about new requests and other
+# events.
+EMAIL_ADMIN_LIST = {{ email_admin_list }} or ['admin@{{ hostname }}']
 EMAIL_SENDER = '{{ from_email }}'
 EMAIL_TICKET_SYSTEM_ADDRESS = 'help@{{ hostname }}'
 EMAIL_DIRECTOR_EMAIL_ADDRESS = 'director@{{ hostname }}'
@@ -139,6 +141,17 @@ CONSTANCE_REDIS_CONNECTION = {
     'password': '{{ redis_passwd }}',
 }
 
+#------------------------------------------------------------------------------
+# django debug toolbar settings
+#------------------------------------------------------------------------------
+
+# IP addresses other than 127.0.0.1 that can view the django debug toolbar.
+EXTRA_INTERNAL_IPS = [
+{% for ip in debug_toolbar_ips %}
+    '{{ ip }}',
+{% endfor %}
+]
+
 #------------------------------------------------------------------------------
 # django-flags settings
 #------------------------------------------------------------------------------