Move fourfront from pytest 2.9 to 3.10 (ALT)

.ebextensions/13_run_npm.config

@@ -31,6 +31,7 @@ files:
 
             echo "Finished NPM build in 99_run_npm." >> /var/log/deploy.log
             echo "Finished NPM build in 99_run_npm."
+
             echo "Restarting Apache"
 
             sudo service httpd restart

.gitignore

@@ -1,4 +1,5 @@
 .coverage
+*.srctr*
 coverage.xml
 beanstalk.cfg
 /coverage
@@ -16,6 +17,7 @@ beanstalk.cfg
 /develop-eggs/
 /downloads/
 /eggs/
+/ped_files_data/
 /.eggs/
 /extends/
 /jsconfig.json
@@ -66,8 +68,17 @@ elasticsearch-*.deb
 # Emacs backup files
 *~
 
+# PyCharm metadata
+.idea/
+
 # Virtual environments
+# Conventions vary, including env, venv, or longer names like cgap_env,
+# but typically they end in "env".
 *env/
 
-# PyCharm
-.idea/
+# Saved configurations from Elastic beanstalk that have been downloaded.
+.elasticbeanstalk/saved_configs/
+.elasticbeanstalk/app_versions/
+
+# Used for some kinds of debugging in dcicutils, snovault, cgap & ff.
+DEBUGLOG-*

Makefile

@@ -1,3 +1,5 @@
+SHELL=/bin/bash
+
 clean:  # clear node modules, eggs, npm build stuff
 	make clean-python-caches
 	make clean-npm-caches
@@ -38,7 +40,7 @@ macpoetry-install:  # Same as 'poetry install' except that on OSX Catalina, an e
 
 configure:  # does any pre-requisite installs
 	pip install --upgrade pip
-	pip install poetry==1.0.10  # pinned to avoid build problems we cannot fix in pyproject.toml
+	pip install poetry==1.1.4  # poetry latest as of 1/25/2021 seemed to work but apparantly does not
 
 build:  # builds
 	make configure
@@ -93,11 +95,17 @@ deploy2:  # spins up waittress to serve the application
 	pserve development.ini
 
 psql-dev:  # starts psql with the url after 'sqlalchemy.url =' in development.ini
-	@psql `grep 'sqlalchemy[.]url =' development.ini | sed -E 's/^.* = (.*)/\1/'`
+	@scripts/psql-start dev
+
+psql-test:  # starts psql with a url constructed from data in 'ps aux'.
+	@scripts/psql-start test
 
-kibana-start:
+kibana-start:  # starts a dev version of kibana (default port)
 	scripts/kibana-start
 
+kibana-start-test:  # starts a test version of kibana (port chosen for active tests)
+	scripts/kibana-start test
+
 kibana-stop:
 	scripts/kibana-stop
 
@@ -112,16 +120,36 @@ clean-python:
 	pip freeze | xargs pip uninstall -y
 
 test:
-	bin/test -vv --timeout=200 -m "working and not performance"
+	make test-npm
+	make test-unit
+
+retest:
+	bin/test  -vv --last-failed
 
 test-any:
 	bin/test -vv --timeout=200
 
+test-npm:
+	bin/test -vv --timeout=200 -m "working and not manual and not integratedx and not performance and not broken and not broken_locally and not sloppy and not indexing"
+
+test-unit:
+	bin/test -vv --timeout=200 -m "working and not manual and not integratedx and not performance and not broken and not broken_locally and not sloppy and indexing"
+
+test-performance:
+	bin/test -vv --timeout=200 -m "working and not manual and not integratedx and performance and not broken and not broken_locally and not sloppy"
+
+test-integrated:
+	bin/test -vv --timeout=200 -m "working and not manual and (integrated or integratedx) and not performance and not broken and not broken_locally and not sloppy"
+
+travis-test:  # Actually, we don't normally use this. Instead the GA workflow sets up two parallel tests.
+	make travis-test-npm
+	make travis-test-unit
+
 travis-test-npm:  # Note this only does the 'not indexing' tests
-	bin/test -vv --force-flaky --max-runs=3 --timeout=400 -m "working and not performance and not indexing and not action_fail" --aws-auth --durations=10 --cov src/encoded --es search-fourfront-testing-6-8-kncqa2za2r43563rkcmsvgn2fq.us-east-1.es.amazonaws.com:443
+	bin/test -vv --force-flaky --max-runs=3 --timeout=400 -m "working and not manual and not integratedx and not performance and not broken and not broken_remotely and not sloppy and not indexing" --aws-auth --durations=10 --cov src/encoded --es search-fourfront-testing-6-8-kncqa2za2r43563rkcmsvgn2fq.us-east-1.es.amazonaws.com:443
 
 travis-test-unit:  # Note this does the 'indexing' tests
-	bin/test -vv --force-flaky --max-runs=3 --timeout=400 -m "working and not performance and indexing and not action_fail" --aws-auth --durations=10 --cov src/encoded --es search-fourfront-testing-6-8-kncqa2za2r43563rkcmsvgn2fq.us-east-1.es.amazonaws.com:443
+	bin/test -vv --force-flaky --max-runs=3 --timeout=400 -m "working and not manual and not integratedx and not performance and not broken and not broken_remotely and not sloppy and indexing" --aws-auth --durations=10 --cov src/encoded --es search-fourfront-testing-6-8-kncqa2za2r43563rkcmsvgn2fq.us-east-1.es.amazonaws.com:443
 
 update:  # updates dependencies
 	poetry update
@@ -140,11 +168,14 @@ info:
 	   $(info - Use 'make configure' to install poetry. You should not have to do this directly.)
 	   $(info - Use 'make deploy1' to spin up postgres/elasticsearch and load inserts.)
 	   $(info - Use 'make deploy2' to spin up the application server.)
-	   $(info - Use 'make psql-dev' to start psql on data associated with an active 'make deploy1'.)
-	   $(info - Use 'make kibana-start' to start kibana, and 'make kibana-stop' to stop it.)
+	   $(info - Use 'make kibana-start' to start kibana on the default local ES port, and 'make kibana-stop' to stop it.)
+	   $(info - Use 'make kibana-start-test' to start kibana on the port being used for active testing, and 'make kibana-stop' to stop it.)
 	   $(info - Use 'make kill' to kill postgres and elasticsearch proccesses. Please use with care.)
 	   $(info - Use 'make moto-setup' to install moto, for less flaky tests. Implied by 'make build'.)
 	   $(info - Use 'make npm-setup' to build the front-end. Implied by 'make build'.)
-	   $(info - Use 'make test' to run tests with normal options we use on travis ('-m "working and not performance"').)
+	   $(info - Use 'make psql-dev' to start psql on data associated with an active 'make deploy1'.)
+	   $(info - Use 'make psql-test' to start psql on data associated with an active test.)
+	   $(info - Use 'make retest' to run failing tests from the previous test run.)
+	   $(info - Use 'make test' to run tests with normal options similar to what we use on GitHub Actions.)
 	   $(info - Use 'make test-any' to run tests without marker constraints (i.e., with no '-m' option).)
 	   $(info - Use 'make update' to update dependencies (and the lock file).)

conftest.py

@@ -1,4 +1,6 @@
 import pytest
+import tempfile
+
 
 def pytest_addoption(parser):
     parser.addoption("--es", action="store", default="", dest='es',
@@ -15,3 +17,10 @@ def remote_es(request):
 @pytest.fixture(scope='session')
 def aws_auth(request):
     return request.config.getoption("--aws-auth")
+
+
+def pytest_configure():
+    # This adjustment is important to set the default choice of temporary filenames to a nice short name
+    # because without it some of the filenames we generate end up being too long, and critical functionality
+    # ends up failing. Some socket-related filenames, for example, seem to have length limits. -kmp 5-Jun-2020
+    tempfile.tempdir = '/tmp'