From 3d9d8a5753611383ab2806ecbfecbd84ab98ca66 Mon Sep 17 00:00:00 2001 From: mightqxc Date: Tue, 22 Oct 2024 16:59:52 +0200 Subject: [PATCH 1/7] add base class of spec --- pandacommon/pandautils/base.py | 117 +++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 pandacommon/pandautils/base.py diff --git a/pandacommon/pandautils/base.py b/pandacommon/pandautils/base.py new file mode 100644 index 0000000..edc86a8 --- /dev/null +++ b/pandacommon/pandautils/base.py @@ -0,0 +1,117 @@ +############################## +# Base classes in PanDA/JEDI # +############################## + + +class SpecBase(object): + """ + Base class of specification + """ + # attributes + attributes = () + # attributes which have 0 by default + _zeroAttrs = () + # attributes to force update + _forceUpdateAttrs = () + + # constructor + def __init__(self): + # install attributes + for attr in self.attributes: + self._orig_setattr(attr, None) + # map of changed attributes + self._orig_setattr("_changedAttrs", {}) + + # override __setattr__ to collect the changed attributes + def __setattr__(self, name, value): + oldVal = getattr(self, name) + self._orig_setattr(name, value) + newVal = getattr(self, name) + # collect changed attributes + if oldVal != newVal or name in self._forceUpdateAttrs: + self._changedAttrs[name] = value + + def _orig_setattr(self, name, value): + """ + original setattr method + """ + super().__setattr__(self, name, value) + + def resetChangedList(self): + """ + reset changed attribute list + """ + self._orig_setattr("_changedAttrs", {}) + + def forceUpdate(self, name): + """ + force update the attribute + """ + if name in self.attributes: + self._changedAttrs[name] = getattr(self, name) + + def valuesMap(self, onlyChanged=False): + """ + return map of values + """ + ret = {} + for attr in self.attributes: + # only changed attributes + if onlyChanged: + if attr not in self._changedAttrs: + continue + val = getattr(self, attr) + if val is None: + if attr in self._zeroAttrs: + val = 0 + else: + val = None + ret[f":{attr}"] = val + return ret + + def pack(self, values): + """ + # pack tuple into spec + """ + for i in range(len(self.attributes)): + attr = self.attributes[i] + val = values[i] + self._orig_setattr(attr, val) + + @classmethod + def columnNames(cls, prefix=None): + """ + return column names for INSERT + """ + attr_list = [] + for attr in cls.attributes: + if prefix is not None: + attr_list.append(f"{prefix}.{attr}") + else: + attr_list.append(f"{attr}") + ret = ",".join(attr_list) + return ret + + @classmethod + def bindValuesExpression(cls): + """ + return expression of bind variables for INSERT + """ + attr_list = [] + for attr in cls.attributes: + attr_list.append(f":{attr}") + attrs_str = ",".join(attr_list) + ret = f"VALUES({attrs_str}) " + return ret + + def bindUpdateChangesExpression(self): + """ + return an expression of bind variables for UPDATE to update only changed attributes + """ + attr_list = [] + for attr in self.attributes: + if attr in self._changedAttrs: + attr_list.append(f"{attr}=:{attr}") + attrs_str = ",".join(attr_list) + ret = f"{attrs_str} " + return ret From 5f40115f7c87ef9370253712bba63ce84fe9d482 Mon Sep 17 00:00:00 2001 From: mightqxc Date: Tue, 5 Nov 2024 10:42:06 +0100 Subject: [PATCH 2/7] fix naive_utcnow --- pandacommon/pandautils/PandaUtils.py | 12 ++++++------ pandacommon/pandautils/base.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pandacommon/pandautils/PandaUtils.py b/pandacommon/pandautils/PandaUtils.py index 393905e..4dfdd03 100644 --- a/pandacommon/pandautils/PandaUtils.py +++ b/pandacommon/pandautils/PandaUtils.py @@ -33,17 +33,17 @@ def isLogRotating(before_limit, after_limit): return False -def aware_utcnow() -> datetime: +def aware_utcnow() -> datetime.datetime.: """ Return the current UTC date and time, with tzinfo timezone.utc Returns: datetime: current UTC date and time, with tzinfo timezone.utc """ - return datetime.now(timezone.utc) + return datetime.datetime.now(timezone.utc) -def aware_utcfromtimestamp(timestamp: float) -> datetime: +def aware_utcfromtimestamp(timestamp: float) -> datetime.datetime: """ Return the local date and time, with tzinfo timezone.utc, corresponding to the POSIX timestamp @@ -53,10 +53,10 @@ def aware_utcfromtimestamp(timestamp: float) -> datetime: Returns: datetime: current UTC date and time, with tzinfo timezone.utc """ - return datetime.fromtimestamp(timestamp, timezone.utc) + return datetime.datetime.fromtimestamp(timestamp, timezone.utc) -def naive_utcnow() -> datetime: +def naive_utcnow() -> datetime.datetime: """ Return the current UTC date and time, without tzinfo @@ -66,7 +66,7 @@ def naive_utcnow() -> datetime: return aware_utcnow().replace(tzinfo=None) -def naive_utcfromtimestamp(timestamp: float) -> datetime: +def naive_utcfromtimestamp(timestamp: float) -> datetime.datetime: """ Return the local date and time, without tzinfo, corresponding to the POSIX timestamp diff --git a/pandacommon/pandautils/base.py b/pandacommon/pandautils/base.py index edc86a8..ddc4f0f 100644 --- a/pandacommon/pandautils/base.py +++ b/pandacommon/pandautils/base.py @@ -71,7 +71,7 @@ def valuesMap(self, onlyChanged=False): def pack(self, values): """ - # pack tuple into spec + pack tuple into spec """ for i in range(len(self.attributes)): attr = self.attributes[i] From 370ae77e0245e73e68ec9bf1ad1e2aca954516e0 Mon Sep 17 00:00:00 2001 From: mightqxc Date: Tue, 5 Nov 2024 10:43:20 +0100 Subject: [PATCH 3/7] fix --- pandacommon/pandautils/PandaUtils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandacommon/pandautils/PandaUtils.py b/pandacommon/pandautils/PandaUtils.py index 4dfdd03..6050118 100644 --- a/pandacommon/pandautils/PandaUtils.py +++ b/pandacommon/pandautils/PandaUtils.py @@ -33,7 +33,7 @@ def isLogRotating(before_limit, after_limit): return False -def aware_utcnow() -> datetime.datetime.: +def aware_utcnow() -> datetime.datetime: """ Return the current UTC date and time, with tzinfo timezone.utc From 8dcc8747d27bc7ff51f7cbf098967937e3330fec Mon Sep 17 00:00:00 2001 From: mightqxc Date: Tue, 5 Nov 2024 10:45:40 +0100 Subject: [PATCH 4/7] fix --- pandacommon/pandautils/PandaUtils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandacommon/pandautils/PandaUtils.py b/pandacommon/pandautils/PandaUtils.py index 6050118..4461041 100644 --- a/pandacommon/pandautils/PandaUtils.py +++ b/pandacommon/pandautils/PandaUtils.py @@ -40,7 +40,7 @@ def aware_utcnow() -> datetime.datetime: Returns: datetime: current UTC date and time, with tzinfo timezone.utc """ - return datetime.datetime.now(timezone.utc) + return datetime.datetime.now(datetime.timezone.utc) def aware_utcfromtimestamp(timestamp: float) -> datetime.datetime: @@ -53,7 +53,7 @@ def aware_utcfromtimestamp(timestamp: float) -> datetime.datetime: Returns: datetime: current UTC date and time, with tzinfo timezone.utc """ - return datetime.datetime.fromtimestamp(timestamp, timezone.utc) + return datetime.datetime.fromtimestamp(timestamp, datetime.timezone.utc) def naive_utcnow() -> datetime.datetime: From 2de601caf989cad1c7f6bfea4a62766aafa443a0 Mon Sep 17 00:00:00 2001 From: mightqxc Date: Tue, 5 Nov 2024 13:42:53 +0100 Subject: [PATCH 5/7] add back seq in base spec --- pandacommon/pandautils/base.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/pandacommon/pandautils/base.py b/pandacommon/pandautils/base.py index ddc4f0f..af5139b 100644 --- a/pandacommon/pandautils/base.py +++ b/pandacommon/pandautils/base.py @@ -13,6 +13,8 @@ class SpecBase(object): _zeroAttrs = () # attributes to force update _forceUpdateAttrs = () + # mapping between sequence and attr + _seqAttrMap = {} # constructor def __init__(self): @@ -35,7 +37,7 @@ def _orig_setattr(self, name, value): """ original setattr method """ - super().__setattr__(self, name, value) + super().__setattr__(name, value) def resetChangedList(self): """ @@ -50,12 +52,15 @@ def forceUpdate(self, name): if name in self.attributes: self._changedAttrs[name] = getattr(self, name) - def valuesMap(self, onlyChanged=False): + def valuesMap(self, useSeq=False, onlyChanged=False): """ return map of values """ ret = {} for attr in self.attributes: + # use sequence + if useSeq and attr in self._seqAttrMap: + continue # only changed attributes if onlyChanged: if attr not in self._changedAttrs: @@ -93,13 +98,16 @@ def columnNames(cls, prefix=None): return ret @classmethod - def bindValuesExpression(cls): + def bindValuesExpression(cls, useSeq=True): """ return expression of bind variables for INSERT """ attr_list = [] for attr in cls.attributes: - attr_list.append(f":{attr}") + if useSeq and attr in cls._seqAttrMap: + attr_list.append(f"{cls._seqAttrMap[attr]}") + else: + attr_list.append(f":{attr}") attrs_str = ",".join(attr_list) ret = f"VALUES({attrs_str}) " return ret From f2c4264fb805f5e0d307c423d8e0f55ecd9f4fcc Mon Sep 17 00:00:00 2001 From: mightqxc Date: Thu, 7 Nov 2024 23:30:54 +0100 Subject: [PATCH 6/7] add itertools batched --- pandacommon/pandautils/PandaUtils.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pandacommon/pandautils/PandaUtils.py b/pandacommon/pandautils/PandaUtils.py index 4461041..e0d3f72 100644 --- a/pandacommon/pandautils/PandaUtils.py +++ b/pandacommon/pandautils/PandaUtils.py @@ -1,4 +1,5 @@ import datetime +import itertools import pytz @@ -77,3 +78,18 @@ def naive_utcfromtimestamp(timestamp: float) -> datetime.datetime: datetime: current UTC date and time, without tzinfo """ return aware_utcfromtimestamp(timestamp).replace(tzinfo=None) + + +def batched(iterable, n, *, strict=False): + """ + Batch data from the iterable into tuples of length n. The last batch may be shorter than n + If strict is true, will raise a ValueError if the final batch is shorter than n + Note this function is for Python <= 3.11 as it mimics itertools.batched() in Python 3.13 + """ + if n < 1: + raise ValueError("n must be at least one") + iterator = iter(iterable) + while batch := tuple(itertools.islice(iterator, n)): + if strict and len(batch) != n: + raise ValueError("batched(): incomplete batch") + yield batch \ No newline at end of file From 5bb2eb39395e0d8961214ce0a2b9e34c7f15639c Mon Sep 17 00:00:00 2001 From: mightqxc Date: Thu, 7 Nov 2024 23:50:40 +0100 Subject: [PATCH 7/7] v0.1.3 --- PandaPkgInfo.py | 2 +- pandacommon/pandautils/PandaUtils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/PandaPkgInfo.py b/PandaPkgInfo.py index 158a7b1..c5b5d24 100644 --- a/PandaPkgInfo.py +++ b/PandaPkgInfo.py @@ -1 +1 @@ -release_version = "0.1.2" +release_version = "0.1.3" diff --git a/pandacommon/pandautils/PandaUtils.py b/pandacommon/pandautils/PandaUtils.py index e0d3f72..b66b73f 100644 --- a/pandacommon/pandautils/PandaUtils.py +++ b/pandacommon/pandautils/PandaUtils.py @@ -92,4 +92,4 @@ def batched(iterable, n, *, strict=False): while batch := tuple(itertools.islice(iterator, n)): if strict and len(batch) != n: raise ValueError("batched(): incomplete batch") - yield batch \ No newline at end of file + yield batch