Merge pull request DarkEnergySurvey#94 from DarkEnergySurvey/tests

Candidate for v1.8.0

.github/workflows/python-package.yml

@@ -32,23 +32,21 @@ jobs:
         conda info
     - name: Create conda environment
       run: |
-        conda create -q -n env python=${{ matrix.python-version }} numpy scipy matplotlib astropy healpy pyyaml emcee fitsio corner -c conda-forge -c kadrlica
+        conda create -y -q -n env python=${{ matrix.python-version }} numpy scipy matplotlib astropy healpy pyyaml emcee fitsio corner -c conda-forge -c kadrlica
+        # Add UGALIDIR to environment
+        conda env config vars set UGALIDIR=$HOME/.ugali -n env
     - name: Install package
       run: |
         source activate env
-        export UGALIDIR="$HOME/.ugali"
-        python setup.py -q install --isochrones --catalogs
+        conda env config vars list
+        python setup.py -q install --isochrones --catalogs --tests
     - name: Lint with flake8
       if: ${{ true }}
       run: |
         source activate env
         conda install -q flake8 -c conda-forge
         # stop the build if there are Python syntax errors or undefined names
         flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-    - name: Install test data
-      run: |
-        wget https://github.com/DarkEnergySurvey/ugali/releases/download/v1.7.0/ugali-test-data.tar.gz -O ugali-test-data.tar.gz
-        tar -xzf ugali-test-data.tar.gz
     - name: Test with nose
       run: |
         source activate env

README.md

@@ -50,7 +50,7 @@ The `ugali` source code is distributed with several auxiliary libraries for isoc
 
 ```
 cd $UGALIDIR
-wget https://github.com/DarkEnergySurvey/ugali/releases/download/v1.7.0/ugali-des-bressan2012.tar.gz
+wget https://github.com/DarkEnergySurvey/ugali/releases/download/v1.8.0/ugali-des-bressan2012.tar.gz
 tar -xzf ugali-des-bressan2012.tar.gz
 ```
 

setup.py

@@ -24,25 +24,27 @@ def find_packages():
 HERE = os.path.abspath(os.path.dirname(__file__))
 URL = 'https://github.com/DarkEnergySurvey/ugali'
 DESC = "Ultra-faint galaxy likelihood toolkit."
-LONG_DESC = "See %s"%URL
+LONG_DESC = "%s\n%s"%(DESC,URL)
 CLASSIFIERS = """\
-Development Status :: 3 - Alpha
+Development Status :: 4 - Beta
 Intended Audience :: Science/Research
 Intended Audience :: Developers
 License :: OSI Approved :: MIT License
 Natural Language :: English
 Operating System :: MacOS :: MacOS X
 Operating System :: POSIX :: Linux
-Programming Language :: Python :: 2.7
+Programming Language :: Python :: 2
+Programming Language :: Python :: 3
 Topic :: Scientific/Engineering
 Topic :: Scientific/Engineering :: Astronomy
 Topic :: Scientific/Engineering :: Physics
 """
 
-RELEASE = URL+'/releases/download/v1.7.0'
+RELEASE_URL = URL+'/releases/download/v1.8.0'
 UGALIDIR = os.getenv("UGALIDIR","$HOME/.ugali")
 ISOSIZE = "~1MB" 
 CATSIZE = "~20MB"
+TSTSIZE = "~1MB"
 # Could find file size dynamically, but it's a bit slow...
 # int(urllib.urlopen(ISOCHRONES).info().getheaders("Content-Length")[0])/1024**2
 SURVEYS = ['des','ps1','sdss','lsst']
@@ -62,7 +64,7 @@ def read(self, size):
     def progress_bar(count, block_size, total_size):
         block = 100*block_size/float(total_size)
         progress = count*block
-        if progress % 1 < 1.01*block:
+        if progress % 5 < 1.01*block:
             msg = '\r[{:51}] ({:d}%)'.format(int(progress//2)*'='+'>',int(progress))
             sys.stdout.write(msg)
             sys.stdout.flush()
@@ -77,7 +79,7 @@ class TarballCommand(distutils.cmd.Command,object):
          'force installation (overwrite any existing files)')
         ]
     boolean_options = ['force']
-    release = RELEASE
+    release_url = RELEASE_URL
     _tarball = None
     _dirname = None
 
@@ -114,7 +116,7 @@ def install_tarball(self, tarball):
             os.makedirs(self.ugali_dir)
         os.chdir(self.ugali_dir)
 
-        url = os.path.join(self.release,tarball)
+        url = os.path.join(self.release_url,tarball)
 
         print("downloading %s..."%url)
         if urlopen(url).getcode() >= 400:
@@ -158,6 +160,12 @@ class CatalogCommand(TarballCommand):
     _tarball = 'ugali-catalogs.tar.gz'
     _dirname = 'catalogs'
 
+class TestsCommand(TarballCommand):
+    """ Command for downloading catalog files """
+    description = "install test data"
+    _tarball = 'ugali-test-data.tar.gz'
+    _dirname = 'testdata'
+
 class IsochroneCommand(TarballCommand):
     """ Command for downloading isochrone files """
     description = "install isochrone files"
@@ -223,6 +231,7 @@ class install(_install):
     user_options = _install.user_options + [
         ('isochrones',None,"install isochrone files (%s)"%ISOSIZE),
         ('catalogs',None,"install catalog files (%s)"%CATSIZE),
+        ('tests',None,"install test data (%s)"%TSTSIZE),
         ('ugali-dir=',None,"install file directory [default: %s]"%UGALIDIR),
     ]
     boolean_options = _install.boolean_options + ['isochrones','catalogs']
@@ -232,6 +241,7 @@ def initialize_options(self):
         self.ugali_dir = os.path.expandvars(UGALIDIR)
         self.isochrones = False
         self.catalogs = False
+        self.tests = False
 
     def run(self):
         # run superclass install
@@ -246,7 +256,10 @@ def run(self):
 
         if self.catalogs: 
             self.install_catalogs()
-
+
+        if self.tests:
+            self.install_tests()
+
     def install_isochrones(self):
         """
         Call to isochrone install command:
@@ -266,10 +279,21 @@ def install_catalogs(self):
         cmd_obj.force = self.force
         if self.ugali_dir: cmd_obj.ugali_dir = self.ugali_dir
         self.run_command('catalogs')
-
+
+    def install_tests(self):
+        """
+        Call to catalog install command:
+        http://stackoverflow.com/a/24353921/4075339
+        """
+        cmd_obj = self.distribution.get_command_obj('tests')
+        cmd_obj.force = self.force
+        if self.ugali_dir: cmd_obj.ugali_dir = self.ugali_dir
+        self.run_command('tests')
+
 CMDCLASS = versioneer.get_cmdclass()
 CMDCLASS['isochrones'] = IsochroneCommand
 CMDCLASS['catalogs'] = CatalogCommand
+CMDCLASS['tests'] = TestsCommand
 CMDCLASS['install'] = install
 
 setup(
@@ -278,17 +302,18 @@ def install_catalogs(self):
     cmdclass=CMDCLASS,
     url=URL,
     author='Keith Bechtol & Alex Drlica-Wagner',
-    author_email='bechtol@kicp.uchicago.edu, [email protected]',
+    author_email='bechtol@wisc.edu, [email protected]',
     scripts = [],
     install_requires=[
+        'astropy',
+        'matplotlib',
         'numpy >= 1.9.0',
         'scipy >= 0.14.0',
         'healpy >= 1.6.0',
         'fitsio >= 0.9.10',
         'emcee >= 2.1.0',
         'corner >= 1.0.0',
         'pyyaml >= 3.10',
-        # Add astropy, fitsio, matplotlib, ...
     ],
     packages=find_packages(),
     description=DESC,

tests/config.yaml

@@ -27,7 +27,7 @@ coords:
 
 catalog:
   # Color always defined as mag_1 - mag_2
-  dirname: ./healpix                    # directory of catalog files
+  dirname: ${UGALIDIR}/testdata/healpix          # directory of catalog files
   basename: "catalog_hpx%04i.fits"      # catalog file basename format
   objid_field       : COADD_OBJECT_ID   # unique object identifier
   lon_field         : RA                # name of longitude field
@@ -45,24 +45,19 @@ catalog:
   selection         : "(np.abs(self.data['WAVG_SPREAD_MODEL_I']) < 0.003)"
 
 data: 
-  dirname: ./raw
+  dirname: ${UGALIDIR}/raw
   script : ./ugali/preprocess/database.py
   survey : des
   release: y3a2
   density: ./density/density_hpx%04i.fits
   footprint: ./maps/footprint_nside4096_equ.fits.gz
 
 mask:
-  dirname    : ./mask
+  dirname    : ${UGALIDIR}/testdata/mask
   basename_1 : "maglim_g_hpx%04i.fits"
   basename_2 : "maglim_r_hpx%04i.fits"
   minimum_solid_angle: 0.1 # deg^2
 
-mangle:
-  dirname    : ./maps
-  filename_1 : 'y3a2_g_o.4096_t.32768_maglim_EQU.fits.gz'
-  filename_2 : 'y3a2_r_o.4096_t.32768_maglim_EQU.fits.gz'
-
 #ADW: Depricated in favor of 'binning'
 color:
   min   : &cmin -0.5

tests/test_config.py

@@ -15,12 +15,12 @@ def test_config():
     np.testing.assert_equal(len(config.filenames),768)
 
     np.testing.assert_equal(config.filenames['pix'].compressed()[0],687)
-    np.testing.assert_equal(config.filenames['catalog'].compressed()[0],
-                            './healpix/catalog_hpx0687.fits')
-    np.testing.assert_equal(config.filenames['mask_1'].compressed()[0],
-                            './mask/maglim_g_hpx0687.fits')
-    np.testing.assert_equal(config.filenames['mask_2'].compressed()[0],
-                            './mask/maglim_r_hpx0687.fits')
+    catfile = os.path.basename(config.filenames['catalog'].compressed()[0])
+    np.testing.assert_equal(catfile,'catalog_hpx0687.fits')
+    maskfile = os.path.basename(config.filenames['mask_1'].compressed()[0])
+    np.testing.assert_equal(maskfile,'maglim_g_hpx0687.fits')
+    maskfile = os.path.basename(config.filenames['mask_2'].compressed()[0])
+    np.testing.assert_equal(maskfile,'maglim_r_hpx0687.fits')
 
     np.testing.assert_equal(config.likefile,'./scan/scan_%08i_%s.fits')
     np.testing.assert_equal(config.mergefile,'./scan/merged_scan.fits')

tests/test_observation.py

@@ -44,8 +44,13 @@ def test_catalog():
     """ Test ugali.observation.catalog """
     import ugali.observation.roi
     import ugali.observation.catalog
+    import ugali.utils.config
 
-    filename='healpix/catalog_hpx0687.fits'
+    # Get catalog filename
+    config = ugali.utils.config.Config(CONFIG)
+    filename = config.filenames['catalog'].compressed()[0]
+
+    # Create catalog from filename
     catalog = ugali.observation.catalog.Catalog(CONFIG,filenames=filename)
 
     roi = ugali.observation.roi.ROI(CONFIG, LON, LAT)

ugali/observation/mask.py

@@ -592,7 +592,7 @@ def backgroundCMD(self, catalog, mode='cloud-in-cells', weights=None):
         index_color = np.arange(len(self.roi.centers_color))
         # Add the cumulative leakage back into the last bin of the CMD
         leakage = (cmd_background * ~observable).sum(axis=0)
-        cmd_background[[index_mag,index_color]] += leakage
+        cmd_background[(index_mag,index_color)] += leakage
         # Zero out all non-observable bins
         cmd_background *= observable
 

ugali/utils/config.py

@@ -172,86 +172,6 @@ def write(self, filename):
             raise Exception('Unrecognized config format: %s'%ext)
         writer.close()
 
-    def _createFilenames(self,pixels=None):
-        """
-        Create a masked records array of all filenames for the given set of
-        pixels and store the existence of those files in the mask values.
-
-        Examples:
-        f = getFilenames([1,2,3])
-        # All possible catalog files
-        f['catalog'].data
-        # All existing catalog files
-        f['catalog'][~f.mask['catalog']]
-        # or
-        f['catalog'].compressed()
-        # All missing mask_1 files
-        f['mask_1'][f.mask['mask_1']]
-        # Pixels where all files exist
-        f['pix'][~f.mask['pix']]
-
-        Parameters:
-        -----------
-        pixels : If pixels is None, grab all pixels of 'nside_catalog'.
-
-        Returns:
-        --------
-        recarray : pixels and mask value
-        """
-        nside_catalog = self['coords']['nside_catalog']
-
-        if pixels is not None:
-            pixels = [pixels] if np.isscalar(pixels) else pixels
-        else:
-            pixels = np.arange(hp.nside2npix(nside_catalog))   
-
-        npix = len(pixels)
-
-        catalog_dir = self['catalog']['dirname']
-        catalog_base = self['catalog']['basename']
-
-        mask_dir = self['mask']['dirname']
-        mask_base_1 = self['mask']['basename_1']
-        mask_base_2 = self['mask']['basename_2']
-
-        data = np.ma.empty(npix,dtype=[('pix',int), ('catalog',object), 
-                                       ('mask_1',object), ('mask_2',object)])
-        mask = np.ma.empty(npix,dtype=[('pix',bool), ('catalog',bool), 
-                                       ('mask_1',bool), ('mask_2',bool)])
-        for ii,pix in enumerate(pixels):
-            if pix is None:
-                # DEPRECTATED: ADW 2018-06-17
-                # This is not really being used anymore
-                raise ValueError('pix cannot be None')
-                catalog = os.path.join(catalog_dir,catalog_base)
-                mask_1 = os.path.join(mask_dir,mask_base_1)
-                mask_2 = os.path.join(mask_dir,mask_base_2)
-            else:
-                catalog = os.path.join(catalog_dir,catalog_base%pix)
-                mask_1 = os.path.join(mask_dir,mask_base_1%pix)
-                mask_2 = os.path.join(mask_dir,mask_base_2%pix)
-            data[ii]['pix'] = pix if pix is not None else -1
-            data[ii]['catalog'] = catalog
-            data[ii]['mask_1']  = mask_1
-            data[ii]['mask_2']  = mask_2
-
-            mask[ii]['catalog'] = not os.path.exists(catalog)
-            mask[ii]['mask_1']  = not os.path.exists(mask_1)
-            mask[ii]['mask_2']  = not os.path.exists(mask_2)
-
-        for name in ['catalog','mask_1','mask_2']:
-            if np.all(mask[name]): logger.warn("All '%s' files masked"%name)
-
-        # mask 'pix' if all files not present
-        mask['pix'] = mask['catalog'] | mask['mask_1'] | mask['mask_2']
-
-        if np.all(mask['pix']): logger.warn("All pixels masked")
-
-        #return np.ma.mrecords.MaskedArray(data, mask, fill_value=[-1,None,None,None])
-        #return np.ma.mrecords.MaskedArray(data, mask, fill_value=[-1,'','',''])
-        return np.ma.MaskedArray(data, mask, fill_value=[-1,'','',''])
-
-
     def _createFilenames(self):
         """
         Create a masked records array of all filenames for the given set of
@@ -269,11 +189,17 @@ def _createFilenames(self):
         npix = hp.nside2npix(nside_catalog)
         pixels = np.arange(npix)
 
-        catalog_dir = self['catalog']['dirname']
+        catalog_dir = os.path.expandvars(self['catalog']['dirname'])
+        if not os.path.isdir(catalog_dir):
+            msg = "Directory does not exist: %s"%catalog_dir
+            raise IOError(msg)
         catalog_base = self['catalog']['basename']
         catalog_path = os.path.join(catalog_dir,catalog_base)
 
-        mask_dir    = self['mask']['dirname']
+        mask_dir    = os.path.expandvars(self['mask']['dirname'])
+        if not os.path.isdir(mask_dir):
+            msg = "Directory does not exist: %s"%mask_dir
+            raise IOError(msg)
         mask_base_1 = self['mask']['basename_1']
         mask_base_2 = self['mask']['basename_2']
         mask_path_1 = os.path.join(mask_dir,mask_base_1)

ugali/utils/fileio.py

@@ -121,7 +121,7 @@ def load_header(kwargs):
         if not keys: keys = hdr.keys()
         return {k:hdr[k] for k in keys}
     except Exception as e:
-        logger.error("Failed to load file: %(filename)s"%kwargs)
+        logger.error("Failed to load header from file: %(filename)s"%kwargs)
         raise(e)
 
 def load_headers(filenames,multiproc=False,**kwargs):