From b602c681d46de6f7c62fb2e3cc860a783f86429e Mon Sep 17 00:00:00 2001 From: Brianna Smart Date: Wed, 20 Sep 2023 11:42:13 -0700 Subject: [PATCH] Update avro schemas with docstrings and add update tool --- .github/workflows/build.yml | 3 + python/lsst/alert/packet/__init__.py | 1 + .../packet/schema/6/0/lsst.v6_0.alert.avsc | 23 + .../schema/6/0/lsst.v6_0.diaForcedSource.avsc | 68 ++ .../6/0/lsst.v6_0.diaNondetectionLimit.avsc | 23 + .../schema/6/0/lsst.v6_0.diaObject.avsc | 698 +++++++++++++ .../schema/6/0/lsst.v6_0.diaSource.avsc | 916 ++++++++++++++++++ .../packet/schema/6/0/lsst.v6_0.ssObject.avsc | 476 +++++++++ python/lsst/alert/packet/updateSchema.py | 204 ++++ setup.cfg | 1 + 10 files changed, 2413 insertions(+) create mode 100644 python/lsst/alert/packet/schema/6/0/lsst.v6_0.alert.avsc create mode 100644 python/lsst/alert/packet/schema/6/0/lsst.v6_0.diaForcedSource.avsc create mode 100644 python/lsst/alert/packet/schema/6/0/lsst.v6_0.diaNondetectionLimit.avsc create mode 100644 python/lsst/alert/packet/schema/6/0/lsst.v6_0.diaObject.avsc create mode 100644 python/lsst/alert/packet/schema/6/0/lsst.v6_0.diaSource.avsc create mode 100644 python/lsst/alert/packet/schema/6/0/lsst.v6_0.ssObject.avsc create mode 100644 python/lsst/alert/packet/updateSchema.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 10e7a04..0943bba 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,6 +14,9 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + - name: Install yaml needed for build + run: | + pip install pyyaml - name: Build and install wheel run: | pip install virtualenv diff --git a/python/lsst/alert/packet/__init__.py b/python/lsst/alert/packet/__init__.py index f99163a..a7df15d 100644 --- a/python/lsst/alert/packet/__init__.py +++ b/python/lsst/alert/packet/__init__.py @@ -23,3 +23,4 @@ from .schema import * from .schemaRegistry import * from .simulate import * +from .updateSchema import * diff --git a/python/lsst/alert/packet/schema/6/0/lsst.v6_0.alert.avsc b/python/lsst/alert/packet/schema/6/0/lsst.v6_0.alert.avsc new file mode 100644 index 0000000..19bcd10 --- /dev/null +++ b/python/lsst/alert/packet/schema/6/0/lsst.v6_0.alert.avsc @@ -0,0 +1,23 @@ +{ + "namespace": "lsst.v6_0", + "type": "record", + "name": "alert", + "doc": "sample avro alert schema v6.0", + "fields": [ + {"name": "alertId", "type": "long", "doc": "unique alert identifer"}, + {"name": "diaSource", "type": "lsst.v6_0.diaSource"}, + {"name": "prvDiaSources", "type": ["null", { + "type": "array", + "items": "lsst.v6_0.diaSource"}], "default": null}, + {"name": "prvDiaForcedSources", "type": ["null", { + "type": "array", + "items": "lsst.v6_0.diaForcedSource"}], "default": null}, + {"name": "prvDiaNondetectionLimits", "type": ["null", { + "type": "array", + "items": "lsst.v6_0.diaNondetectionLimit"}], "default": null}, + {"name": "diaObject", "type": ["null", "lsst.v6_0.diaObject"], "default": null}, + {"name": "ssObject", "type": ["null", "lsst.v6_0.ssObject"], "default": null}, + {"name": "cutoutDifference", "type": ["null", "bytes"], "default": null}, + {"name": "cutoutTemplate", "type": ["null", "bytes"], "default": null} + ] +} diff --git a/python/lsst/alert/packet/schema/6/0/lsst.v6_0.diaForcedSource.avsc b/python/lsst/alert/packet/schema/6/0/lsst.v6_0.diaForcedSource.avsc new file mode 100644 index 0000000..aeb6400 --- /dev/null +++ b/python/lsst/alert/packet/schema/6/0/lsst.v6_0.diaForcedSource.avsc @@ -0,0 +1,68 @@ +{ + "type": "record", + "namespace": "lsst.v6_0", + "name": "diaForcedSource", + "fields": [ + { + "doc": "Unique id.", + "name": "diaForcedSourceId", + "type": "long" + }, + { + "doc": "Id of the DiaObject that this DiaForcedSource was associated with.", + "name": "diaObjectId", + "type": "long" + }, + { + "doc": "Id of the ccdVisit where this forcedSource was measured.", + "name": "ccdVisitId", + "type": "long" + }, + { + "default": null, + "doc": "Point Source model flux.", + "name": "psfFlux", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Uncertainty of psfFlux.", + "name": "psfFluxErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "x position at which psfFlux has been measured.", + "name": "x", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "y position at which psfFlux has been measured.", + "name": "y", + "type": [ + "null", + "float" + ] + }, + { + "doc": "Flags, bitwise OR tbd", + "name": "flags", + "type": "long" + }, + { + "doc": "Effective mid-visit time for this diaForcedSource, expressed as Modified Julian Date, International Atomic Time.", + "name": "midpointMjdTai", + "type": "double" + } + ] +} \ No newline at end of file diff --git a/python/lsst/alert/packet/schema/6/0/lsst.v6_0.diaNondetectionLimit.avsc b/python/lsst/alert/packet/schema/6/0/lsst.v6_0.diaNondetectionLimit.avsc new file mode 100644 index 0000000..9e40419 --- /dev/null +++ b/python/lsst/alert/packet/schema/6/0/lsst.v6_0.diaNondetectionLimit.avsc @@ -0,0 +1,23 @@ +{ + "type": "record", + "namespace": "lsst.v6_0", + "name": "diaNondetectionLimit", + "fields": [ + { + "name": "ccdVisitId", + "type": "long" + }, + { + "name": "midpointMjdTai", + "type": "double" + }, + { + "name": "band", + "type": "string" + }, + { + "name": "diaNoise", + "type": "float" + } + ] +} \ No newline at end of file diff --git a/python/lsst/alert/packet/schema/6/0/lsst.v6_0.diaObject.avsc b/python/lsst/alert/packet/schema/6/0/lsst.v6_0.diaObject.avsc new file mode 100644 index 0000000..d76f27d --- /dev/null +++ b/python/lsst/alert/packet/schema/6/0/lsst.v6_0.diaObject.avsc @@ -0,0 +1,698 @@ +{ + "type": "record", + "namespace": "lsst.v6_0", + "name": "diaObject", + "fields": [ + { + "doc": "Unique identifier of this DiaObject.", + "name": "diaObjectId", + "type": "long" + }, + { + "doc": "Right ascension coordinate of the position of the object at time radecMjdTai.", + "name": "ra", + "type": "double" + }, + { + "default": null, + "doc": "Uncertainty of ra.", + "name": "raErr", + "type": [ + "null", + "float" + ] + }, + { + "doc": "Declination coordinate of the position of the object at time radecMjdTai.", + "name": "dec", + "type": "double" + }, + { + "default": null, + "doc": "Uncertainty of dec.", + "name": "decErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance between ra and dec.", + "name": "ra_dec_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Time at which the object was at a position ra/dec, expressed as Modified Julian Date, International Atomic Time.", + "name": "radecMjdTai", + "type": [ + "null", + "double" + ] + }, + { + "default": null, + "doc": "Proper motion in right ascension.", + "name": "pmRa", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Uncertainty of pmRa.", + "name": "pmRaErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Proper motion of declination.", + "name": "pmDec", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Uncertainty of pmDec.", + "name": "pmDecErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Parallax.", + "name": "parallax", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Uncertainty of parallax.", + "name": "parallaxErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance of pmRa and pmDec.", + "name": "pmRa_pmDec_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance of pmRa and parallax.", + "name": "pmRa_parallax_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance of pmDec and parallax.", + "name": "pmDec_parallax_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Natural log of the likelihood of the linear proper motion parallax fit.", + "name": "pmParallaxLnL", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Chi^2 static of the model fit.", + "name": "pmParallaxChi2", + "type": [ + "null", + "float" + ] + }, + { + "doc": "The number of data points used to fit the model.", + "name": "pmParallaxNdata", + "type": "int" + }, + { + "default": null, + "doc": "Weighted mean point-source model magnitude for u filter.", + "name": "u_psfFluxMean", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Standard error of u_psfFluxMean.", + "name": "u_psfFluxMeanErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Standard deviation of the distribution of u_psfFlux.", + "name": "u_psfFluxSigma", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Chi^2 statistic for the scatter of u_psfFlux around u_psfFluxMean.", + "name": "u_psfFluxChi2", + "type": [ + "null", + "float" + ] + }, + { + "doc": "The number of data points used to compute u_psfFluxChi2.", + "name": "u_psfFluxNdata", + "type": "int" + }, + { + "default": null, + "doc": "Weighted mean forced photometry flux for u filter.", + "name": "u_fpFluxMean", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Standard error of u_fpFluxMean.", + "name": "u_fpFluxMeanErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Standard deviation of the distribution of u_fpFlux.", + "name": "u_fpFluxSigma", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Weighted mean point-source model magnitude for g filter.", + "name": "g_psfFluxMean", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Standard error of g_psfFluxMean.", + "name": "g_psfFluxMeanErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Standard deviation of the distribution of g_psfFlux.", + "name": "g_psfFluxSigma", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Chi^2 statistic for the scatter of g_psfFlux around g_psfFluxMean.", + "name": "g_psfFluxChi2", + "type": [ + "null", + "float" + ] + }, + { + "doc": "The number of data points used to compute g_psfFluxChi2.", + "name": "g_psfFluxNdata", + "type": "int" + }, + { + "default": null, + "doc": "Weighted mean forced photometry flux for g filter.", + "name": "g_fpFluxMean", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Standard error of g_fpFluxMean.", + "name": "g_fpFluxMeanErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Standard deviation of the distribution of g_fpFlux.", + "name": "g_fpFluxSigma", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Weighted mean point-source model magnitude for r filter.", + "name": "r_psfFluxMean", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Standard error of r_psfFluxMean.", + "name": "r_psfFluxMeanErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Standard deviation of the distribution of r_psfFlux.", + "name": "r_psfFluxSigma", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Chi^2 statistic for the scatter of r_psfFlux around r_psfFluxMean.", + "name": "r_psfFluxChi2", + "type": [ + "null", + "float" + ] + }, + { + "doc": "The number of data points used to compute r_psfFluxChi2.", + "name": "r_psfFluxNdata", + "type": "int" + }, + { + "default": null, + "doc": "Weighted mean forced photometry flux for r filter.", + "name": "r_fpFluxMean", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Standard error of r_fpFluxMean.", + "name": "r_fpFluxMeanErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Standard deviation of the distribution of r_fpFlux.", + "name": "r_fpFluxSigma", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Weighted mean point-source model magnitude for i filter.", + "name": "i_psfFluxMean", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Standard error of i_psfFluxMean.", + "name": "i_psfFluxMeanErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Standard deviation of the distribution of i_psfFlux.", + "name": "i_psfFluxSigma", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Chi^2 statistic for the scatter of i_psfFlux around i_psfFluxMean.", + "name": "i_psfFluxChi2", + "type": [ + "null", + "float" + ] + }, + { + "doc": "The number of data points used to compute i_psfFluxChi2.", + "name": "i_psfFluxNdata", + "type": "int" + }, + { + "default": null, + "doc": "Weighted mean forced photometry flux for i filter.", + "name": "i_fpFluxMean", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Standard error of i_fpFluxMean.", + "name": "i_fpFluxMeanErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Standard deviation of the distribution of i_fpFlux.", + "name": "i_fpFluxSigma", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Weighted mean point-source model magnitude for z filter.", + "name": "z_psfFluxMean", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Standard error of z_psfFluxMean.", + "name": "z_psfFluxMeanErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Standard deviation of the distribution of z_psfFlux.", + "name": "z_psfFluxSigma", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Chi^2 statistic for the scatter of z_psfFlux around z_psfFluxMean.", + "name": "z_psfFluxChi2", + "type": [ + "null", + "float" + ] + }, + { + "doc": "The number of data points used to compute z_psfFluxChi2.", + "name": "z_psfFluxNdata", + "type": "int" + }, + { + "default": null, + "doc": "Weighted mean forced photometry flux for z filter.", + "name": "z_fpFluxMean", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Standard error of z_fpFluxMean.", + "name": "z_fpFluxMeanErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Standard deviation of the distribution of z_fpFlux.", + "name": "z_fpFluxSigma", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Weighted mean point-source model magnitude for y filter.", + "name": "y_psfFluxMean", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Standard error of y_psfFluxMean.", + "name": "y_psfFluxMeanErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Standard deviation of the distribution of y_psfFlux.", + "name": "y_psfFluxSigma", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Chi^2 statistic for the scatter of y_psfFlux around y_psfFluxMean.", + "name": "y_psfFluxChi2", + "type": [ + "null", + "float" + ] + }, + { + "doc": "The number of data points used to compute y_psfFluxChi2.", + "name": "y_psfFluxNdata", + "type": "int" + }, + { + "default": null, + "doc": "Weighted mean forced photometry flux for y filter.", + "name": "y_fpFluxMean", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Standard error of y_fpFluxMean.", + "name": "y_fpFluxMeanErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Standard deviation of the distribution of y_fpFlux.", + "name": "y_fpFluxSigma", + "type": [ + "null", + "float" + ] + }, + { + "doc": "Id of the closest nearby object.", + "name": "nearbyObj1", + "type": "long" + }, + { + "default": null, + "doc": "Distance to nearbyObj1.", + "name": "nearbyObj1Dist", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Natural log of the probability that the observed diaObject is the same as the nearbyObj1.", + "name": "nearbyObj1LnP", + "type": [ + "null", + "float" + ] + }, + { + "doc": "Id of the second-closest nearby object.", + "name": "nearbyObj2", + "type": "long" + }, + { + "default": null, + "doc": "Distance to nearbyObj2.", + "name": "nearbyObj2Dist", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Natural log of the probability that the observed diaObject is the same as the nearbyObj2.", + "name": "nearbyObj2LnP", + "type": [ + "null", + "float" + ] + }, + { + "doc": "Id of the third-closest nearby object.", + "name": "nearbyObj3", + "type": "long" + }, + { + "default": null, + "doc": "Distance to nearbyObj3.", + "name": "nearbyObj3Dist", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Natural log of the probability that the observed diaObject is the same as the nearbyObj3.", + "name": "nearbyObj3LnP", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Mean of the u band flux errors.", + "name": "u_psfFluxErrMean", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Mean of the g band flux errors.", + "name": "g_psfFluxErrMean", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Mean of the r band flux errors.", + "name": "r_psfFluxErrMean", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Mean of the i band flux errors.", + "name": "i_psfFluxErrMean", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Mean of the z band flux errors.", + "name": "z_psfFluxErrMean", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Mean of the y band flux errors.", + "name": "y_psfFluxErrMean", + "type": [ + "null", + "float" + ] + }, + { + "doc": "Flags, bitwise OR tbd.", + "name": "flags", + "type": "long" + } + ] +} \ No newline at end of file diff --git a/python/lsst/alert/packet/schema/6/0/lsst.v6_0.diaSource.avsc b/python/lsst/alert/packet/schema/6/0/lsst.v6_0.diaSource.avsc new file mode 100644 index 0000000..567a3b3 --- /dev/null +++ b/python/lsst/alert/packet/schema/6/0/lsst.v6_0.diaSource.avsc @@ -0,0 +1,916 @@ +{ + "type": "record", + "namespace": "lsst.v6_0", + "name": "diaSource", + "fields": [ + { + "doc": "Unique identifier of this DiaSource.", + "name": "diaSourceId", + "type": "long" + }, + { + "doc": "Id of the ccdVisit where this diaSource was measured.", + "name": "ccdVisitId", + "type": "long" + }, + { + "default": null, + "doc": "Id of the diaObject this source was associated with, if any. If not, it is set to NULL (each diaSource will be associated with either a diaObject or ssObject).", + "name": "diaObjectId", + "type": [ + "null", + "long" + ] + }, + { + "default": null, + "doc": "Id of the ssObject this source was associated with, if any. If not, it is set to NULL (each diaSource will be associated with either a diaObject or ssObject).", + "name": "ssObjectId", + "type": [ + "null", + "long" + ] + }, + { + "default": null, + "doc": "Id of the parent diaSource this diaSource has been deblended from, if any.", + "name": "parentDiaSourceId", + "type": [ + "null", + "long" + ] + }, + { + "doc": "Effective mid-visit time for this diaSource, expressed as Modified Julian Date, International Atomic Time.", + "name": "midpointMjdTai", + "type": "double" + }, + { + "doc": "Right ascension coordinate of the center of this diaSource.", + "name": "ra", + "type": "double" + }, + { + "default": null, + "doc": "Uncertainty of ra.", + "name": "raErr", + "type": [ + "null", + "float" + ] + }, + { + "doc": "Declination coordinate of the center of this diaSource.", + "name": "dec", + "type": "double" + }, + { + "default": null, + "doc": "Uncertainty of dec.", + "name": "decErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance between ra and dec.", + "name": "ra_dec_Cov", + "type": [ + "null", + "float" + ] + }, + { + "doc": "x position computed by a centroiding algorithm.", + "name": "x", + "type": "float" + }, + { + "default": null, + "doc": "Uncertainty of x.", + "name": "xErr", + "type": [ + "null", + "float" + ] + }, + { + "doc": "y position computed by a centroiding algorithm.", + "name": "y", + "type": "float" + }, + { + "default": null, + "doc": "Uncertainty of y.", + "name": "yErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance between x and y.", + "name": "x_y_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Flux in a 12 pixel radius aperture on the difference image.", + "name": "apFlux", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Estimated uncertainty of apFlux.", + "name": "apFluxErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "The signal-to-noise ratio at which this source was detected in the difference image.", + "name": "snr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Flux for Point Source model. Note this actually measures the flux difference between the template and the visit image.", + "name": "psfFlux", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Uncertainty of psfFlux.", + "name": "psfFluxErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Right ascension coordinate of centroid for point source model.", + "name": "psfRa", + "type": [ + "null", + "double" + ] + }, + { + "default": null, + "doc": "Uncertainty of psfRa.", + "name": "psfRaErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Declination coordinate of centroid for point source model.", + "name": "psfDec", + "type": [ + "null", + "double" + ] + }, + { + "default": null, + "doc": "Uncertainty of psfDec.", + "name": "psfDecErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance between psfFlux and psfRa.", + "name": "psfFlux_psfRa_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance between psfFlux and psfDec.", + "name": "psfFlux_psfDec_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance between psfRa and psfDec.", + "name": "psfRa_psfDec_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Natural log likelihood of the observed data given the point source model.", + "name": "psfLnL", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Chi^2 statistic of the point source model fit.", + "name": "psfChi2", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "The number of data points (pixels) used to fit the point source model.", + "name": "psfNdata", + "type": [ + "null", + "int" + ] + }, + { + "default": null, + "doc": "Flux for a trailed source model. Note this actually measures the flux difference between the template and the visit image.", + "name": "trailFlux", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Uncertainty of trailFlux.", + "name": "trailFluxErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Right ascension coordinate of centroid for trailed source model.", + "name": "trailRa", + "type": [ + "null", + "double" + ] + }, + { + "default": null, + "doc": "Uncertainty of trailRa.", + "name": "trailRaErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Declination coordinate of centroid for trailed source model.", + "name": "trailDec", + "type": [ + "null", + "double" + ] + }, + { + "default": null, + "doc": "Uncertainty of trailDec.", + "name": "trailDecErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Maximum likelihood fit of trail length.", + "name": "trailLength", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Uncertainty of trailLength.", + "name": "trailLengthErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Maximum likelihood fit of the angle between the meridian through the centroid and the trail direction (bearing).", + "name": "trailAngle", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Uncertainty of trailAngle.", + "name": "trailAngleErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance of trailFlux and trailRa.", + "name": "trailFlux_trailRa_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance of trailFlux and trailDec.", + "name": "trailFlux_trailDec_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance of trailFlux and trailLength", + "name": "trailFlux_trailLength_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance of trailFlux and trailAngle", + "name": "trailFlux_trailAngle_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance of trailRa and trailDec.", + "name": "trailRa_trailDec_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance of trailRa and trailLength.", + "name": "trailRa_trailLength_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance of trailRa and trailAngle.", + "name": "trailRa_trailAngle_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance of trailDec and trailLength.", + "name": "trailDec_trailLength_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance of trailDec and trailAngle.", + "name": "trailDec_trailAngle_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance of trailLength and trailAngle", + "name": "trailLength_trailAngle_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Natural log likelihood of the observed data given the trailed source model.", + "name": "trailLnL", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Chi^2 statistic of the trailed source model fit.", + "name": "trailChi2", + "type": [ + "null", + "float" + ] + }, + { + "doc": "The number of data points (pixels) used to fit the trailed source model.", + "name": "trailNdata", + "type": "int" + }, + { + "default": null, + "doc": "Maximum likelihood value for the mean absolute flux of the two lobes for a dipole model.", + "name": "dipoleMeanFlux", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Uncertainty of dipoleMeanFlux.", + "name": "dipoleMeanFluxErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Maximum likelihood value for the difference of absolute fluxes of the two lobes for a dipole model.", + "name": "dipoleFluxDiff", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Uncertainty of dipoleFluxDiff.", + "name": "dipoleFluxDiffErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": " Right ascension coordinate of centroid for dipole model.", + "name": "dipoleRa", + "type": [ + "null", + "double" + ] + }, + { + "default": null, + "doc": "Uncertainty of dipoleRa.", + "name": "dipoleRaErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": " Declination coordinate of centroid for dipole model.", + "name": "dipoleDec", + "type": [ + "null", + "double" + ] + }, + { + "default": null, + "doc": "Uncertainty of dipoleDec.", + "name": "dipoleDecErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Maximum likelihood value for the lobe separation in dipole model.", + "name": "dipoleLength", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Uncertainty of dipoleLength.", + "name": "dipoleLengthErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Maximum likelihood fit of the angle between the meridian through the centroid and the dipole direction (bearing, from negative to positive lobe).", + "name": "dipoleAngle", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Uncertainty of dipoleAngle.", + "name": "dipoleAngleErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance of dipoleMeanFlux and dipoleFluxDiff.", + "name": "dipoleMeanFlux_dipoleFluxDiff_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance of dipoleMeanFlux and dipoleRa.", + "name": "dipoleMeanFlux_dipoleRa_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance of dipoleMeanFlux and dipoleDec.", + "name": "dipoleMeanFlux_dipoleDec_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance of dipoleMeanFlux and dipoleLength.", + "name": "dipoleMeanFlux_dipoleLength_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance of dipoleMeanFlux and dipoleAngle.", + "name": "dipoleMeanFlux_dipoleAngle_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance of dipoleFluxDiff and dipoleRa.", + "name": "dipoleFluxDiff_dipoleRa_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance of dipoleFluxDiff and dipoleDec.", + "name": "dipoleFluxDiff_dipoleDec_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance of dipoleFluxDiff and dipoleLength.", + "name": "dipoleFluxDiff_dipoleLength_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance of dipoleFluxDiff and dipoleAngle.", + "name": "dipoleFluxDiff_dipoleAngle_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance of dipoleRa and dipoleDec.", + "name": "dipoleRa_dipoleDec_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance of dipoleRa and dipoleLength.", + "name": "dipoleRa_dipoleLength_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance of dipoleRa and dipoleAngle.", + "name": "dipoleRa_dipoleAngle_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance of dipoleDec and dipoleLength.", + "name": "dipoleDec_dipoleLength_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance of dipoleDec and dipoleAngle.", + "name": "dipoleDec_dipoleAngle_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance of dipoleLength and dipoleAngle.", + "name": "dipoleLength_dipoleAngle_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Natural log likelihood of the observed data given the dipole source model.", + "name": "dipoleLnL", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Chi^2 statistic of the model fit.", + "name": "dipoleChi2", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "The number of data points (pixels) used to fit the model.", + "name": "dipoleNdata", + "type": [ + "null", + "int" + ] + }, + { + "default": null, + "doc": "Calibrated flux for Point Source model centered on radec but measured on the difference of snaps comprising this visit.", + "name": "snapDiffFlux", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Estimated uncertainty of snapDiffFlux.", + "name": "snapDiffFluxErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Estimated sky background at the position (centroid) of the object.", + "name": "fpBkgd", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Estimated uncertainty of fpBkgd.", + "name": "fpBkgdErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Adaptive second moment of the source intensity.", + "name": "ixx", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Uncertainty of ixx.", + "name": "ixxErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Adaptive second moment of the source intensity.", + "name": "iyy", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Uncertainty of iyy.", + "name": "iyyErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Adaptive second moment of the source intensity.", + "name": "ixy", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Uncertainty of ixy.", + "name": "ixyErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance of ixx and iyy.", + "name": "ixx_iyy_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance of ixx and ixy.", + "name": "ixx_ixy_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Covariance of iyy and ixy.", + "name": "iyy_ixy_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Adaptive second moment for the PSF.", + "name": "ixxPSF", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Adaptive second moment for the PSF.", + "name": "iyyPSF", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Adaptive second moment for the PSF.", + "name": "ixyPSF", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "A measure of extendedness, Computed using a combination of available moments and model fluxes or from a likelihood ratio of point/trailed source models (exact algorithm TBD). extendedness = 1 implies a high degree of confidence that the source is extended. extendedness = 0 implies a high degree of confidence that the source is point-like.", + "name": "extendedness", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "A measure of reliability, computed using information from the source and image characterization, as well as the information on the Telescope and Camera system (e.g., ghost maps, defect maps, etc.).", + "name": "reliability", + "type": [ + "null", + "float" + ] + }, + { + "doc": "Flags, bitwise OR tbd.", + "name": "flags", + "type": "long" + }, + { + "default": null, + "doc": "Object determined to be a dipole.", + "name": "isDipole", + "type": [ + "null", + "boolean" + ] + }, + { + "default": null, + "doc": "", + "name": "bboxSize", + "type": [ + "null", + "long" + ] + } + ] +} \ No newline at end of file diff --git a/python/lsst/alert/packet/schema/6/0/lsst.v6_0.ssObject.avsc b/python/lsst/alert/packet/schema/6/0/lsst.v6_0.ssObject.avsc new file mode 100644 index 0000000..46a1984 --- /dev/null +++ b/python/lsst/alert/packet/schema/6/0/lsst.v6_0.ssObject.avsc @@ -0,0 +1,476 @@ +{ + "type": "record", + "namespace": "lsst.v6_0", + "name": "ssObject", + "fields": [ + { + "doc": "Unique identifier.", + "name": "ssObjectId", + "type": "long" + }, + { + "default": null, + "doc": "The date the LSST first linked and submitted the discovery observations to the MPC. May be NULL if not an LSST discovery. The date format will follow general LSST conventions (MJD TAI, at the moment).", + "name": "discoverySubmissionDate", + "type": [ + "null", + "double" + ] + }, + { + "default": null, + "doc": "The time of the first LSST observation of this object (could be precovered) as Modified Julian Date, International Atomic Time.", + "name": "firstObservationDate", + "type": [ + "null", + "double" + ] + }, + { + "default": null, + "doc": "Arc of LSST observations.", + "name": "arc", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Number of LSST observations of this object.", + "name": "numObs", + "type": [ + "null", + "int" + ] + }, + { + "default": null, + "doc": "Minimum orbit intersection distance to Earth.", + "name": "MOID", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "True anomaly of the MOID point.", + "name": "MOIDTrueAnomaly", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Ecliptic longitude of the MOID point.", + "name": "MOIDEclipticLongitude", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "DeltaV at the MOID point.", + "name": "MOIDDeltaV", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Best fit absolute magnitude (u band).", + "name": "u_H", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Best fit G12 slope parameter (u band).", + "name": "u_G12", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Uncertainty of H (u band).", + "name": "u_HErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Uncertainty of G12 (u band).", + "name": "u_G12Err", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "H-G12 covariance (u band).", + "name": "u_H_u_G12_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Chi^2 statistic of the phase curve fit (u band).", + "name": "u_Chi2", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "The number of data points used to fit the phase curve (u band).", + "name": "u_Ndata", + "type": [ + "null", + "int" + ] + }, + { + "default": null, + "doc": "Best fit absolute magnitude (g band).", + "name": "g_H", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Best fit G12 slope parameter (g band).", + "name": "g_G12", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Uncertainty of H (g band).", + "name": "g_HErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Uncertainty of G12 (g band).", + "name": "g_G12Err", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "H-G12 covariance (g band).", + "name": "g_H_g_G12_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Chi^2 statistic of the phase curve fit (g band).", + "name": "g_Chi2", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "The number of data points used to fit the phase curve (g band).", + "name": "g_Ndata", + "type": [ + "null", + "int" + ] + }, + { + "default": null, + "doc": "Best fit absolute magnitude (r band).", + "name": "r_H", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Best fit G12 slope parameter (r band).", + "name": "r_G12", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Uncertainty of H (r band).", + "name": "r_HErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Uncertainty of G12 (r band).", + "name": "r_G12Err", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "H-G12 covariance (r band).", + "name": "r_H_r_G12_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Chi^2 statistic of the phase curve fit (r band).", + "name": "r_Chi2", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "The number of data points used to fit the phase curve (r band).", + "name": "r_Ndata", + "type": [ + "null", + "int" + ] + }, + { + "default": null, + "doc": "Best fit absolute magnitude (i band).", + "name": "i_H", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Best fit G12 slope parameter (i band).", + "name": "i_G12", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Uncertainty of H (i band).", + "name": "i_HErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Uncertainty of G12 (i band).", + "name": "i_G12Err", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "H-G12 covariance (i band).", + "name": "i_H_i_G12_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Chi^2 statistic of the phase curve fit (i band).", + "name": "i_Chi2", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "The number of data points used to fit the phase curve (i band).", + "name": "i_Ndata", + "type": [ + "null", + "int" + ] + }, + { + "default": null, + "doc": "Best fit absolute magnitude (z band).", + "name": "z_H", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Best fit G12 slope parameter (z band).", + "name": "z_G12", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Uncertainty of H (z band).", + "name": "z_HErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Uncertainty of G12 (z band).", + "name": "z_G12Err", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "H-G12 covariance (z band).", + "name": "z_H_z_G12_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Chi^2 statistic of the phase curve fit (z band).", + "name": "z_Chi2", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "The number of data points used to fit the phase curve (z band).", + "name": "z_Ndata", + "type": [ + "null", + "int" + ] + }, + { + "default": null, + "doc": "Best fit absolute magnitude (y band).", + "name": "y_H", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Best fit G12 slope parameter (y band).", + "name": "y_G12", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Uncertainty of H (y band).", + "name": "y_HErr", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Uncertainty of G12 (y band).", + "name": "y_G12Err", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "H-G12 covariance (y band).", + "name": "y_H_y_G12_Cov", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "Chi^2 statistic of the phase curve fit (y band).", + "name": "y_Chi2", + "type": [ + "null", + "float" + ] + }, + { + "default": null, + "doc": "The number of data points used to fit the phase curve (y band).", + "name": "y_Ndata", + "type": [ + "null", + "int" + ] + }, + { + "default": null, + "doc": "median `extendedness` value from the DIASource.", + "name": "medianExtendedness", + "type": [ + "null", + "float" + ] + }, + { + "doc": "Flags, bitwise OR tbd.", + "name": "flags", + "type": "long" + } + ] +} \ No newline at end of file diff --git a/python/lsst/alert/packet/updateSchema.py b/python/lsst/alert/packet/updateSchema.py new file mode 100644 index 0000000..0872e2c --- /dev/null +++ b/python/lsst/alert/packet/updateSchema.py @@ -0,0 +1,204 @@ +# This file is part of alert_packet. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (https://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import argparse +import os +import fastavro +import yaml +import json +from lsst.alert.packet import get_schema_root, SchemaRegistry + +__all__ = ['update_schema'] + +def write_schema(schema, version, path): + + updated_schema_loc=path[0:-3] + version[0]+'/'+version[2] + + if not os.path.exists(updated_schema_loc): + os.mkdir(updated_schema_loc[0:-2]) + os.mkdir(updated_schema_loc) + + updated_schema_name = schema['namespace'] + '.' + schema['name'] + '.avsc' + + with open(updated_schema_loc + '/' + updated_schema_name, "w") as f: + json.dump(schema, f, indent=2) + +def add_namespace(schema): + + schema_names = schema['name'].split(".") + schema['name'] = schema_names[2] + schema['namespace'] = schema_names[0] + "." + schema_names[1] + schema = dict(sorted(schema.items(), reverse=True)) + + return schema + + +def populate_fields(apdb): + """Make a dictionary of fields to populate the avro schema. At present, a + number of rules must be included to ensure certain fields are excluded. + + Parameters + ---------- + apdb: `dict` + The name of the schema as a string. E.G. diaSource. + """ + field_dictionary_array = [] + for column in apdb['columns']: + # The diaObject apdb has a lot of extra fields that are not in the schema and they need to be ignored. + if (column['name'] != 'validityStart') and ( + column['name'] != 'validityEnd') and not ("Periodic" in column[ + 'name']) and not "max" in column['name'] and not "min" in column[ + 'name'] and not "Science" in column['name'] and not "Percentile" in column[ + 'name'] and not "Max" in column['name'] and not "Min" in column[ + 'name'] and not "science" in column['name'] and not "LowzGal" in column[ + 'name'] and not "MAD" in column['name'] and not "Skew" in column[ + 'name'] and not "Intercept" in column['name'] and not "Slope" in column[ + 'name'] and not "Stetson" in column['name'] and not "lastNonForcedSource" in column[ + 'name'] and not "nDiaSources" in column['name'] and not "ExtObj" in column[ + 'name'] and not "time_" in column['name'] and not "band" in column[ + 'name'] and not "Time" in column['name']: + + # Check if a column is nullable. If it is, it needs a default. + if 'nullable' in column: + if column['nullable'] == False: + # Check if a column has a description, if so, include "doc" + if 'description' in column: + field = {"name": column['name'], + "type": column["datatype"], + "doc": column["description"]} + field_dictionary_array.append(field) + else: + field = {"name": column['name'], + "type": column["datatype"], "doc": ""} + field_dictionary_array.append(field) + else: # nullable == True + if 'description' in column: + field = {"name": column['name'], + "type": ["null", str(column["datatype"])], + "doc": column["description"], "default": None} + field_dictionary_array.append(field) + else: + field = {"name": column['name'], + "type": ["null", str(column["datatype"])], + "doc": "", "default": None} + field_dictionary_array.append(field) + else: # nullable not in columns (nullable == True) + if 'description' in column: + field = {"name": column['name'], + "type": ["null", str(column["datatype"])], + "doc": column["description"], "default": None} + field_dictionary_array.append(field) + else: + field = {"name": column['name'], + "type": ["null", str(column["datatype"])], + "doc": "", "default": None} + field_dictionary_array.append(field) + + return field_dictionary_array + +def create_schema(name, field_dictionary_array, version): + """ Create a schema using the field dictionary. fastavro will automatically + take the name and namespace and put them as one, so the name should just be + the schema name and the namespace needs to be created separately. The + fastavro keys also need to be removed from the schema. + + Parameters + ---------- + name: `string` + The name of the schema as a string. E.G. diaSource. + + field_dictionary_array: 'np.array' + An array containing dictionary entries for the individual fields. + + version: 'string' + The version number of the schema + """ + name = name[0:2].lower() + name[2:] + schema = fastavro.parse_schema({ + "name": name, + "type": "record", + "fields": field_dictionary_array + }) + + schema['namespace'] = 'lsst.v' + version + fastavro_keys = list(schema.keys()) + for key in fastavro_keys: + if '__' in key and '__len__' not in key: + schema.pop(key) + + schema = dict(sorted(schema.items(), reverse=True)) + + return schema +def update_schema(apdb_filepath, update_version=False): + """Compare an avro schemas docstrings with the apdb.yaml file. + + If there are no docstrings, add the field. If the docstrings do + not match, update the schema docstrings to the apdb docstrings. + + Once it is updated, write out the new schema. + + Parameters + ---------- + apdb_filepath: `string` + Input string for the apdb.yaml file where the docstrings + will be compared. + + update_version: 'bool' + If True, version will be updated minor version one number. + + """ + + registry = SchemaRegistry.from_filesystem() + + for version in registry.known_versions: + schema_root = get_schema_root() + path = os.path.join(schema_root, *version.split(".")) + + with open(apdb_filepath, 'r') as file: + apdb = yaml.safe_load(file) + + if update_version: + version_name = str(int(version.split(".")[0]) + 1) + "_0" + else: + version_name = version.split(".")[0] + "_0" + + # The first 4 columns in the apdb are the ones we use for alerts + for x in range(0, 4): + + name = apdb['tables'][x]['name'] + field_dictionary = populate_fields(apdb['tables'][x]) + schema = create_schema(name, field_dictionary, version_name) + + write_schema(schema, version_name, path) + + +if __name__ == '__main__': + + parser = argparse.ArgumentParser(description='Update the schema docstrings so that they' + 'match the docstrings in apdb.yaml. Example input:' + 'python3 updateSchema.py ' + '"Path/To/Yaml/sdm_schemas/yml/apdb.yaml" False') + parser.add_argument('apdb_filepath') + parser.add_argument('update_version') + + args = parser.parse_args() + + update_schema(args.apdb_filepath, args.update_version) diff --git a/setup.cfg b/setup.cfg index c4d88da..0a82645 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,6 +16,7 @@ install_requires = fastavro numpy requests + pyyaml packages = lsst.alert.packet lsst.alert.packet.bin