diff --git a/docs/manual.md b/docs/manual.md index 897f2968..83fa3d89 100644 --- a/docs/manual.md +++ b/docs/manual.md @@ -67,7 +67,7 @@ If you do not work in Equinor: ODBC queries may already work for you, although i ### For Equinor users -The Web APIs are queried with the requests package. Requests does not utilize the system certificate store, but instead relies on the certifi bundle. In order to avoid SSL verification errors, we need to either turn off SSL verification (optional input argument `verifySSL=False` for relevant function calls) or, strongly preferred, add the certificate to the certifi bundle. To to this, simply activate the virtual environment where you installed tagreader, and run the following snippet: +The Web APIs are queried with the requests package. Requests does not utilize the system certificate store, but instead relies on the certifi bundle. In order to avoid SSL verification errors, we need to either turn off SSL verification (optional input argument `verifySSL=False` for relevant function calls) or, strongly preferred, add the certificate to the certifi bundle. To do this, simply activate the virtual environment where you installed tagreader, and run the following snippet: ``` python from tagreader.utils import add_statoil_root_certificate diff --git a/docs/quickstart.ipynb b/docs/quickstart.ipynb index 7618fcd4..3eb7c939 100644 --- a/docs/quickstart.ipynb +++ b/docs/quickstart.ipynb @@ -5,7 +5,7 @@ "metadata": {}, "source": [ "## Quickstart\n", - "This document provides a quick demonstration of the basic usage of tagreader. It will show you the steps from importing the package to fetching data and making a plot. Some cells contain links to more details that can be found in the [manual](./manual.ipynb)." + "This document provides a quick demonstration of the basic usage of tagreader. It will show you the steps from importing the package to fetching data and making a plot. Some cells contain links to more details that can be found in the [manual](./manual.md)." ] }, { diff --git a/pytest.ini b/pytest.ini index 2d9748c6..e2b8afad 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,3 @@ [pytest] -mock_use_standalone_module = true junit_family = xunit1 python_files = test_*.py diff --git a/tagreader/cache.py b/tagreader/cache.py index 715ee33b..7ac0051b 100644 --- a/tagreader/cache.py +++ b/tagreader/cache.py @@ -32,7 +32,7 @@ def key_path(self, df, readtype, ts=None): """ name = list(df)[0] if isinstance(df, pd.DataFrame) else df name = safe_tagname(name) - ts = ts.seconds if isinstance(ts, pd.Timedelta) else ts + ts = int(ts.total_seconds()) if isinstance(ts, pd.Timedelta) else ts if readtype != ReaderType.RAW: if ts is None: # Determine sample time by reading interval between first two @@ -122,7 +122,7 @@ def readtype_to_str(rt): def timedelta_to_str(t): if isinstance(t, pd.Timedelta): - return str(t.seconds) + return str(int(t.total_seconds())) return t key = "/" + key.lstrip("/") # Ensure absolute path diff --git a/tagreader/clients.py b/tagreader/clients.py index 999c14fa..d1baf270 100644 --- a/tagreader/clients.py +++ b/tagreader/clients.py @@ -65,7 +65,8 @@ def get_missing_intervals(df, start_time, stop_time, ts, read_type): read_type == ReaderType.RAW ): # Fixme: How to check for completeness for RAW data? return [[start_time, stop_time]] - tvec = pd.date_range(start=start_time, end=stop_time, freq=f"{ts}s") + seconds = int(ts.total_seconds()) + tvec = pd.date_range(start=start_time, end=stop_time, freq=f"{seconds}s") if len(df) == len(tvec): # Short-circuit if dataset is complete return [] values_in_df = tvec.isin(df.index) @@ -292,7 +293,7 @@ def _read_single_tag(self, tag, start_time, stop_time, ts, read_type, cache=None stop_time=time_slice[1], ) missing_intervals = get_missing_intervals( - df, start_time, stop_time, ts.seconds, read_type + df, start_time, stop_time, ts, read_type ) if not missing_intervals: return df.tz_convert(self.tz).sort_index() diff --git a/tagreader/odbc_handlers.py b/tagreader/odbc_handlers.py index ab6e2b41..699d6f87 100644 --- a/tagreader/odbc_handlers.py +++ b/tagreader/odbc_handlers.py @@ -95,11 +95,11 @@ def generate_read_query(tag, mapdef, start_time, stop_time, sample_time, read_ty if read_type != ReaderType.SNAPSHOT: start_time = start_time.tz_convert("UTC") stop_time = stop_time.tz_convert("UTC") - sample_time = sample_time.seconds + seconds = int(sample_time.total_seconds()) if ReaderType.SAMPLED == read_type: - sample_time = 0 + seconds = 0 else: - if sample_time <= 0: + if seconds <= 0: raise NotImplementedError # sample_time = (stop_time-start_time).totalseconds @@ -159,7 +159,7 @@ def generate_read_query(tag, mapdef, start_time, stop_time, sample_time, read_ty if mapdef: query.extend([f'AND FIELD_ID = FT({mapdef["MAP_HistoryValue"]!r})']) if ReaderType.RAW != read_type: - query.extend([f"AND (period = {sample_time*10})"]) + query.extend([f"AND (period = {seconds*10})"]) query.extend( [ f"AND (request = {request_num})", @@ -405,11 +405,11 @@ def generate_read_query( if read_type != ReaderType.SNAPSHOT: start_time = start_time.tz_convert("UTC") stop_time = stop_time.tz_convert("UTC") - sample_time = sample_time.seconds + seconds = int(sample_time.total_seconds()) if ReaderType.SAMPLED == read_type: - sample_time = 0 + seconds = 0 else: - if sample_time <= 0: + if seconds <= 0: pass # Fixme: Not implemented # sample_time = (stop_time-start_time).totalseconds @@ -461,13 +461,13 @@ def generate_read_query( elif ReaderType.SHAPEPRESERVING == read_type: query.extend( [ - f"AND (intervalcount = {int((stop_time-start_time).seconds/sample_time)})" # noqa E501 + f"AND (intervalcount = {int((stop_time-start_time).seconds/seconds)})" # noqa E501 ] ) elif ReaderType.RAW == read_type: pass elif read_type not in [ReaderType.SNAPSHOT, ReaderType.RAW]: - query.extend([f"AND (timestep = '{sample_time}s')"]) + query.extend([f"AND (timestep = '{seconds}s')"]) if ReaderType.SNAPSHOT != read_type: query.extend(["ORDER BY time"]) diff --git a/tagreader/utils.py b/tagreader/utils.py index 8c4058b7..7df216bb 100644 --- a/tagreader/utils.py +++ b/tagreader/utils.py @@ -113,7 +113,7 @@ def add_statoil_root_certificate(): if pem in certifi.contents(): print("Certificate already exists in certifi store. Nothing to do.") break - print("Writing certificate to cacert store.") + print("Writing certificate to certifi store.") cafile = certifi.where() with open(cafile, "ab") as f: f.write(bytes(pem, "ascii")) diff --git a/tagreader/web_handlers.py b/tagreader/web_handlers.py index 7a3150a2..ad5142a8 100644 --- a/tagreader/web_handlers.py +++ b/tagreader/web_handlers.py @@ -508,7 +508,7 @@ def generate_read_query( webid = tag if read_type != ReaderType.SNAPSHOT: - sample_time = sample_time.seconds + seconds = int(sample_time.total_seconds()) get_action = { ReaderType.INT: "interpolated", @@ -542,10 +542,10 @@ def generate_read_query( }.get(read_type, None) if ReaderType.INT == read_type: - params["interval"] = f"{sample_time}s" + params["interval"] = f"{seconds}s" elif summary_type: params["summaryType"] = summary_type - params["summaryDuration"] = f"{sample_time}s" + params["summaryDuration"] = f"{seconds}s" if self._is_summary(read_type): params[ diff --git a/tests/test_AspenHandlerODBC.py b/tests/test_AspenHandlerODBC.py index fbc387a9..03c2d121 100644 --- a/tests/test_AspenHandlerODBC.py +++ b/tests/test_AspenHandlerODBC.py @@ -119,3 +119,22 @@ def test_generate_tag_read_query(read_type): } assert expected[read_type] == res + + +def test_genreadquery_long_sampletime(): + starttime = utils.ensure_datetime_with_tz(START_TIME) + stoptime = utils.ensure_datetime_with_tz(STOP_TIME) + ts = pd.Timedelta(86401, unit="s") + + res = AspenHandlerODBC.generate_read_query( + "thetag", None, starttime, stoptime, ts, ReaderType.INT + ) + + expected = ( + 'SELECT ISO8601(ts) AS "time", value AS "value" FROM history WHERE ' + "name = 'thetag' AND (period = 864010) AND (request = 7) " + "AND (ts BETWEEN '2018-01-17T15:00:00Z' AND '2018-01-17T16:00:00Z') " + "ORDER BY ts" + ) + + assert expected == res diff --git a/tests/test_AspenHandlerREST.py b/tests/test_AspenHandlerREST.py index 5cee31f2..86f00f81 100644 --- a/tests/test_AspenHandlerREST.py +++ b/tests/test_AspenHandlerREST.py @@ -93,7 +93,7 @@ def test_generate_map_query(AspenHandler): def test_generate_tag_read_query(AspenHandler, read_type): start_time = utils.ensure_datetime_with_tz("2020-06-24 17:00:00") stop_time = utils.ensure_datetime_with_tz("2020-06-24 18:00:00") - ts = pd.Timedelta(1, unit="m") + ts = pd.Timedelta(SAMPLE_TIME, unit="s") res = AspenHandler.generate_read_query( "ATCAI", None, start_time, stop_time, ts, getattr(ReaderType, read_type) ) @@ -171,3 +171,20 @@ def test_generate_tag_read_query(AspenHandler, read_type): ), } assert expected[read_type] == res + +def test_genreadquery_long_sampletime(AspenHandler): + start_time = utils.ensure_datetime_with_tz("2020-06-24 17:00:00") + stop_time = utils.ensure_datetime_with_tz("2020-06-24 18:00:00") + ts = pd.Timedelta(86401, unit="s") + + res = AspenHandler.generate_read_query( + "ATCAI", None, start_time, stop_time, ts, ReaderType.INT + ) + expected = ( + '' + "" + "015930108000001593014400000" + "10

86401

3" + ) + + assert expected == res \ No newline at end of file diff --git a/tests/test_PIHandlerODBC.py b/tests/test_PIHandlerODBC.py index 34133085..22023ded 100644 --- a/tests/test_PIHandlerODBC.py +++ b/tests/test_PIHandlerODBC.py @@ -119,3 +119,22 @@ def test_generate_tag_read_query(PIHandler, read_type): ), } assert expected[read_type] == res + + +def test_genreadquery_long_sampletime(PIHandler): + starttime = utils.ensure_datetime_with_tz(START_TIME) + stoptime = utils.ensure_datetime_with_tz(STOP_TIME) + ts = pd.Timedelta(86401, unit="s") + + res = PIHandler.generate_read_query( + "thetag", starttime, stoptime, ts, ReaderType.INT + ) + + expected = ( + "SELECT CAST(value as FLOAT32) AS value, time " + "FROM [piarchive]..[piinterp2] WHERE tag='thetag' " + "AND (time BETWEEN '17-Jan-18 15:00:00' AND '17-Jan-18 16:00:00') " + "AND (timestep = '86401s') ORDER BY time" + ) + + assert expected == res \ No newline at end of file diff --git a/tests/test_PIHandlerREST.py b/tests/test_PIHandlerREST.py index d378ac55..f3ceeb49 100644 --- a/tests/test_PIHandlerREST.py +++ b/tests/test_PIHandlerREST.py @@ -125,3 +125,17 @@ def test_generate_read_query(PIHandler, read_type): # TODO: Move away from test params["selectedFields"] == "Links;Items.Timestamp;Items.Value;Items.Good" ) assert params["maxCount"] == 10000 + +def test_genreadquery_long_sampletime(PIHandler): + starttime = ensure_datetime_with_tz(START_TIME) + stoptime = ensure_datetime_with_tz(STOP_TIME) + ts = pd.Timedelta(86410, unit="s") + + (url, params) = PIHandler.generate_read_query( + PIHandler.tag_to_webid("alreadyknowntag"), + starttime, + stoptime, + ts, + ReaderType.INT, + ) + assert params["interval"] == f"{86410}s" diff --git a/tests/test_cache.py b/tests/test_cache.py index df37da6a..2b9d8636 100644 --- a/tests/test_cache.py +++ b/tests/test_cache.py @@ -78,6 +78,10 @@ def test_match_tag(cache): cache._match_tag("INT/s60/tag1", readtype=ReaderType.INT, ts=60, tagname="tag1") is True ) + assert ( + cache._match_tag("INT/s86401/tag1", readtype=ReaderType.INT, ts=86401, tagname="tag1") + is True + ) assert ( cache._match_tag("INT/s60/tag1", readtype=ReaderType.RAW, ts=60, tagname="tag1") is False diff --git a/tests/test_clients.py b/tests/test_clients.py index 90e6dd2a..17557366 100644 --- a/tests/test_clients.py +++ b/tests/test_clients.py @@ -41,7 +41,7 @@ def test_get_missing_intervals(): df, start_time="2018-01-18 05:00:00", stop_time="2018-01-18 06:00:00", - ts=ts, + ts=pd.Timedelta(ts, unit='s'), read_type=ReaderType.INT, ) assert missing[0] == (idx[2], idx[2])