diff --git a/Procfile b/Procfile index e7237b3..2b28ce1 100644 --- a/Procfile +++ b/Procfile @@ -1 +1 @@ -web: wget -O SIMPLE.db https://raw.githubusercontent.com/SIMPLE-AstroDB/SIMPLE-binary/main/SIMPLE.sqlite && python -m simple_app.app_simple -p $PORT -i '0.0.0.0' -f 'SIMPLE.sqlite' +web: wget -O SIMPLE.sqlite https://raw.githubusercontent.com/SIMPLE-AstroDB/SIMPLE-binary/main/SIMPLE.sqlite && python -m simple_app.app_simple -p $PORT -i '0.0.0.0' -f 'SIMPLE.sqlite' diff --git a/README.md b/README.md index dc6ae51..fb32356 100644 --- a/README.md +++ b/README.md @@ -38,18 +38,13 @@ We also recommend keeping up to date with the repo changes, and most importantly the [astrodbkit2](https://github.com/dr-rodriguez/AstrodbKit2) package: ```bash git pull -pip install git+https://github.com/dr-rodriguez/AstrodbKit2 +pip install -Ur requirements.txt ``` -You can also get the latest copy of the SQLite database binary file with: +You can also get the latest copy of the SQLite database binary file again with: ```bash wget https://raw.githubusercontent.com/SIMPLE-AstroDB/SIMPLE-binary/main/SIMPLE.db ``` -If updating requirements, update them with: -```bash -pip freeze > requirements.txt -``` - ## Apache Config The major requirement for running this program on Apache is [`mod_wsgi`](https://flask.palletsprojects.com/en/2.1.x/deploying/mod_wsgi/) diff --git a/environment.yml b/environment.yml index f3fed5e..7af5aa1 100644 --- a/environment.yml +++ b/environment.yml @@ -1,138 +1,9 @@ name: simple channels: - - http://ssb.stsci.edu/astroconda - defaults dependencies: - - _libgcc_mutex=0.1 - - _openmp_mutex=5.1 - - asttokens=2.0.5 - - backcall=0.2.0 - - blas=1.0 - - ca-certificates=2022.10.11 - - decorator=5.1.1 - - executing=0.8.3 - - freetype=2.12.1 - - giflib=5.2.1 - - greenlet=1.1.1 - - intel-openmp=2021.4.0 - - ipython_genutils=0.2.0 - - jedi=0.18.1 - - jpeg=9e - - lcms2=2.12 - - ld_impl_linux-64=2.38 - - lerc=3.0 - - libdeflate=1.8 - - libffi=3.4.2 - - libgcc-ng=11.2.0 - - libgomp=11.2.0 - - libpng=1.6.37 - - libstdcxx-ng=11.2.0 - - libtiff=4.4.0 - - libwebp=1.2.4 - - libwebp-base=1.2.4 - - lz4-c=1.9.3 - - matplotlib-inline=0.1.6 - - mkl=2021.4.0 - - mkl-service=2.4.0 - - mkl_fft=1.3.1 - - mkl_random=1.2.2 - - ncurses=6.3 - - olefile=0.46 - - openssl=1.1.1s - - parso=0.8.3 - - pexpect=4.8.0 - - pickleshare=0.7.5 - - pluggy=1.0.0 - - ptyprocess=0.7.0 - - pure_eval=0.2.2 - - py=1.11.0 - - pygments=2.11.2 - - pyparsing=3.0.9 - - python=3.8.15 - - python-dateutil=2.8.2 - - pytz=2022.1 - - readline=8.2 - - six=1.16.0 - - sqlite=3.40.0 - - stack_data=0.2.0 - - tk=8.6.12 - - tomli=2.0.1 - - tornado=6.2 - - traitlets=5.1.1 - - wcwidth=0.2.5 - - xz=5.2.6 - - yaml=0.2.5 - - zlib=1.2.13 - - zstd=1.5.2 + - python=3.9.16 + - pip - pip: - - asdf==2.10.1 - - asdf-astropy==0.2.2 - - asdf-coordinates-schemas==0.1.0 - - asdf-standard==1.0.3 - - asdf-transform-schemas==0.3.0 - - asdf-wcs-schemas==0.1.1 - - astrodbkit2==0.5 - - astropy==5.2.1 - - astroquery==0.4.6 - - attrs==22.2.0 - - beautifulsoup4==4.9.3 - - bokeh==3.0.2 - - certifi==2022.12.7 - - cffi==1.14.5 - - chardet==4.0.0 - - click==8.1.3 - - contourpy==1.0.6 - - cryptography==40.0.1 - - cycler==0.11.0 - - exceptiongroup==1.1.1 - - flask==2.2.3 - - flask-cors==3.0.10 - - flask-wtf==1.1.1 - - fonttools==4.39.2 - - gwcs==0.18.2 - - html5lib==1.1 - - idna==2.10 - - importlib-metadata==5.1.0 - - importlib-resources==5.10.0 - - iniconfig==2.0.0 - - ipython==8.11.0 - - itsdangerous==2.1.2 - - jeepney==0.6.0 - - jinja2==3.1.2 - - jmespath==1.0.1 - - jsonschema==3.2.0 - - keyring==23.0.1 - - kiwisolver==1.4.4 - - markdown2==2.4.0 - - markupsafe==2.1.2 - - matplotlib==3.7.1 - - ndcube==2.0.3 - - numpy==1.22.4 - - packaging==23.0 - - pandas==1.5.2 - - pillow==9.3.0 - - pip==23.0.1 - - prompt-toolkit==3.0.38 - - pycparser==2.20 - - pyerfa==2.0.0.1 - - pyrsistent==0.17.3 - - pytest==7.2.2 - - pyvo==1.4.1 - - pyyaml==5.4.1 - - requests==2.25.1 - - scipy==1.6.2 - - secretstorage==3.3.1 - - semantic-version==2.8.5 - - setuptools==67.2.0 - - soupsieve==2.2.1 - - specutils==1.9.1 - - sqlalchemy==2.0.7 - - tqdm==4.61.0 - - typing-extensions==4.4.0 - - urllib3==1.26.15 - - webencodings==0.5.1 - - werkzeug==2.2.3 - - wheel==0.40.0 - - wtforms==2.3.3 - - xyzservices==2022.9.0 - - zipp==3.4.1 + - -r requirements.txt + diff --git a/requirements.txt b/requirements.txt index db451d4..bc8b90c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,97 +1,18 @@ -asdf>=2.10.1 -asdf-astropy>=0.2.2 -asdf-coordinates-schemas>=0.1.0 -asdf-standard>=1.0.3 -asdf-transform-schemas>=0.3.0 -asdf-wcs-schemas>=0.1.1 -astrodbkit2>=0.5 -astropy>=5.2.1 -astroquery>=0.4.6 -asttokens>=2.0.5 -attrs>=22.2.0 -backcall>=0.2.0 -beautifulsoup4>=4.9.3 -bokeh>=3.0.2 -certifi>=2022.12.7 -cffi>=1.14.5 -chardet>=4.0.0 -click>=8.1.3 -contourpy>=1.0.6 -cryptography>=39.0.1 -decorator>=5.1.1 -executing>=0.8.3 -exceptiongroup>=1.1.1 -Flask>=2.2.3 -Flask-Cors>=3.0.10 -Flask-WTF>=1.1.1 -greenlet>=1.1.1 -gwcs>=0.18.2 -html5lib>=1.1 -idna>=2.10 -importlib-metadata>=5.1.0 -importlib-resources>=5.10.0 -iniconfig>=2.0.0 -ipython>=8.10.0 -ipython-genutils>=0.2.0 -itsdangerous>=2.1.2 -jedi>=0.18.1 -jeepney>=0.6.0 -Jinja2>=3.1.2 -jmespath>=1.0.1 -jsonschema>=3.2.0 -keyring>=23.0.1 -markdown2>=2.4.0 -MarkupSafe>=2.1.1 -matplotlib-inline>=0.1.6 -mkl-fft>=1.3.1 -mkl-random>=1.2.2 -mkl-service>=2.4.0 -ndcube>=2.0.3 -numpy>=1.22.4 -olefile>=0.46 -packaging>=23.0 -pandas>=1.5.2 -parso>=0.8.3 -pexpect>=4.8.0 -pickleshare>=0.7.5 -Pillow>=9.3.0 -pip>=22.2.2 -pluggy>=1.0.0,<2.0 -prompt-toolkit>=3.0.30,<3.1.0 -ptyprocess>=0.7.0 -pure-eval>=0.2.2 -py>=1.11.0 -pycparser>=2.20 -pyerfa>=2.0.0.1 -Pygments>=2.11.2 -pyparsing>=3.0.9 -pyrsistent>=0.17.3 -pytest>=7.2.2 -python-dateutil>=2.8.2 -pytz>=2022.1 -pyvo>=1.4.1 -PyYAML>=6.0 -requests>=2.25.1 -scipy>=1.6.2 -SecretStorage>=3.3.1 -semantic-version>=2.8.5 -setuptools>=65.5.1,<67.3.0 -six>=1.16.0 -soupsieve>=2.2.1 -specutils>=1.9.1 -SPLAT>=0.32 -SQLAlchemy>=2.0.7 -stack-data>=0.2.0 -tomli>=2.0.1 -tornado>=6.2 -tqdm>=4.61.0 -traitlets>=5.1.1 -typing_extensions>=4.4.0 -urllib3>=1.26.5 -wcwidth>=0.2.5 -webencodings>=0.5.1 -Werkzeug>=2.2.3 -wheel>=0.38.1 -WTForms>=2.3.3 -xyzservices>=2022.9.0 -zipp>=3.4.1 +astrodbkit2>=0.5.1 +astropy>=5.2.0,<6 +bokeh==3.0.2 +flask>=3.0,<4 +flask-cors>=4.0,<5 +flask-wtf>=1.2,<2 +markdown2>=2.4,<3 +multiprocess>=0.70,<1 +numpy>=1.24,<2 +pandas>=2.0,<2.1 +pysqlite3>=0.5,<1 +pytest>=8.0,<9 +requests>=2.31,<3 +specutils>=1.12,<2 +sqlalchemy>=2.0,<3 +tqdm>=4.66,<5 +werkzeug>=3.0,<4 +wtforms>=3.1,<4 diff --git a/simple_app/plots.py b/simple_app/plots.py index da5218c..f9b31f0 100644 --- a/simple_app/plots.py +++ b/simple_app/plots.py @@ -549,7 +549,7 @@ def camd_plot(query: str, everything: Inventory, all_bands: np.ndarray, all_resu this_spectral_type = pd.DataFrame.from_dict(dict(spectral_type_code=[np.nan, ], adopted=[np.nan, ])) # use adopted spectral type if present - this_spectral_type['adopted'] = this_spectral_type.adopted.fillna(False) + this_spectral_type['adopted'] = this_spectral_type.adopted.astype(bool).fillna(False) if not this_spectral_type.adopted.any(): this_spectral_type.loc[0, 'adopted'] = True @@ -571,7 +571,7 @@ def camd_plot(query: str, everything: Inventory, all_bands: np.ndarray, all_resu else: # use adopted parallax, if present - this_parallaxes['adopted'] = this_parallaxes.adopted.fillna(False) + this_parallaxes['adopted'] = this_parallaxes.adopted.astype(bool).fillna(False) if not this_parallaxes.adopted.any(): this_parallaxes.loc[0, 'adopted'] = True this_photometry['parallax'] = this_parallaxes.loc[this_parallaxes.adopted].parallax.iloc[0] diff --git a/simple_app/simports.py b/simple_app/simports.py index 83af683..67f9f57 100644 --- a/simple_app/simports.py +++ b/simple_app/simports.py @@ -36,7 +36,7 @@ from copy import deepcopy # memory control from difflib import get_close_matches # for redirecting bad file paths from io import StringIO, BytesIO, BufferedIOBase # writing files without saving to disk -import multiprocessing as mp # multiprocessing for efficiency +import multiprocess as mp # multiprocessing for efficiency import os # operating system import requests # accessing internet from shutil import copy # copying files diff --git a/simple_app/utils.py b/simple_app/utils.py index d1b6026..0de5f76 100644 --- a/simple_app/utils.py +++ b/simple_app/utils.py @@ -140,8 +140,20 @@ def list_concat(self, key: str, return_markdown: bool = True) -> Union[pd.DataFr """ # construct dataframe for a given key corresponding to a table obj = self.results[key] - df: pd.DataFrame = pd.concat([pd.DataFrame(objrow, index=[i]) - for i, objrow in enumerate(obj)], ignore_index=True) + df_list = [] + + # check each row for NaNs, only append to list if not wholly NaNs + for i, obj_row in enumerate(obj): + df_row = pd.DataFrame(obj_row, index=[i]).dropna(axis=0, how='all') + + if len(df_row): + df_list.append(df_row) + + # create a concatenated dataframe of all the rows in a given table + if len(df_list): + df: pd.DataFrame = pd.concat(df_list, ignore_index=True) + else: + df = pd.DataFrame(columns=list(obj[0].keys())) # switch whether to have a Markdown version of the table, or a normal DataFrame if return_markdown: @@ -727,7 +739,7 @@ def absolute_magnitudes(df: pd.DataFrame, all_bands: np.ndarray) -> pd.DataFrame The output dataframe with absolute mags calculated """ - def pogson_law(m: Union[float, np.ndarray]) -> Union[float, np.ndarray]: + def pogson_law(m: Union[float, pd.Series]) -> Union[float, np.ndarray]: """ Distance modulus equation. Calculates the absolute magnitude only for sources with a positive parallax, otherwise returns a NaN @@ -741,7 +753,10 @@ def pogson_law(m: Union[float, np.ndarray]) -> Union[float, np.ndarray]: _ Absolute magnitude """ - return np.where(df.parallax > 0, m + 5 * np.log10(df.parallax, where=df.parallax > 0) - 10, np.nan) + mask = df.parallax > 0 + _abs_mag = np.full_like(m, fill_value=np.nan) + _abs_mag[mask] = m[mask] + 5 * np.log10(df.parallax[mask]) - 10 + return _abs_mag # create absolute magnitude for each apparent magnitude d_magnitudes: Dict[str, np.ndarray] = {}