Use toml-test v1.3 and add heterogenous array tests

.codesandbox/Dockerfile

@@ -0,0 +1,7 @@
+FROM python:3.10.12
+
+RUN apt-get update && apt-get install -y fish && \
+    pip install -U pip && \
+    pip install poetry && \
+    poetry config virtualenvs.create false && \
+    chsh -s /usr/bin/fish

.codesandbox/tasks.json

@@ -0,0 +1,13 @@
+{
+  // These tasks will run in order when initializing your CodeSandbox project.
+  "setupTasks": [],
+
+  // These tasks can be run from CodeSandbox. Running one will open a log in the app.
+  "tasks": {
+    "poetry update && poetry install": {
+      "name": "poetry update && poetry install",
+      "command": "poetry update && poetry install",
+      "runAtStart": true
+    }
+  }
+}

README.raw.md

@@ -10,8 +10,9 @@ See also: [toml-lang](https://toml.io/en/) and [PEP 680](https://www.python.org/
 
 ## Other reports
 
-- [Tests with `toml-test` v1.0.0](./reports/with_toml-test_v1.0.0.md)
+- [Tests with `toml-test` v1.2.0](./reports/with_toml-test_v1.2.0.md)
 - [Tests with `toml-test` v1.1.0](./reports/with_toml-test_v1.1.0.md)
+- [Tests with `toml-test` v1.0.0](./reports/with_toml-test_v1.0.0.md)
 - [Tests with python 3.11 (`tomllib` included)](./reports/with_python3.11.md)
 
 ## Run your own report

pyproject.toml

@@ -12,11 +12,11 @@ pyparam = "^0.5"
 toml = "^0.10.2"
 rtoml = "^0.9.0"
 # pytomlpp does not support python 3.11 yet
-pytomlpp = {version = "^1.0.11", python = "<3.11"}
+pytomlpp = {version = "^1.0.13", python = "<3.11"}
 tomli = "^2.0.1"
 tomli-w = "^1.0.0"
 qtoml = "^0.3.1"
-tomlkit = "^0.11.6"
+tomlkit = "^0.11.8"
 python-dateutil = "^2.8.2"
 importlib-metadata = "^5.0.0"
 # required by tomlkit, but cannot be inferred by poetry

run-tests.py

@@ -4,7 +4,8 @@
 from pathlib import Path
 
 REPORTS = {
-    "latest": "1.2.0",
+    "latest": "1.3.0",
+    "v1.2.0": "1.2.0",
     "v1.1.0": "1.1.0",
     "v1.0.0": "1.0.0",
 }
@@ -15,7 +16,7 @@ def run_test(comver):
 
     outfile = f"reports/with_toml-test_{comver}.md"
 
-    cmd = f"python -m toml_bench --comver {REPORTS[comver]} --report {outfile}"
+    cmd = f"python -m toml_bench --iter 1000 --comver {REPORTS[comver]} --report {outfile}"
     os.system(cmd)
 
 

toml_bench/__main__.py

@@ -28,7 +28,7 @@ def init_params() -> Params:
     )
     params.add_param(
         "comver",
-        default="1.2.0",
+        default="1.3.0",
         desc="The version of the toml-test to use in compliance tests",
     )
     params.add_param(

toml_bench/cases/hetero_array.py

@@ -0,0 +1,48 @@
+from typing import Any
+from benchwork import BenchCase
+
+
+class BenchCaseDumpsHeteroArray(BenchCase):
+    def run(self) -> Any:
+        v1 = {
+            "v": [1, 1.2, True, "string"]
+        }
+        try:
+            return self.api.dumps(v1)
+        except Exception as e:
+            return e
+
+
+class BenchCaseLoadsHeteroArray(BenchCase):
+
+    def run(self) -> Any:
+        v1 = """
+            v = [1, 1.2, true, "string"]
+        """
+        try:
+            return self.api.loads(v1)
+        except Exception as e1:
+            return e1
+
+
+class BenchCaseDumpsNestedArray(BenchCase):
+    def run(self) -> Any:
+        v1 = {
+            "v": [[1], [1, 2]]
+        }
+        try:
+            return '<pre>' + self.api.dumps(v1) + '</pre>'
+        except Exception as e:
+            return e
+
+
+class BenchCaseLoadsNestedArray(BenchCase):
+
+    def run(self) -> Any:
+        v1 = """
+            v = [[1], [1, 2]]
+        """
+        try:
+            return self.api.loads(v1)
+        except Exception as e1:
+            return e1

toml_bench/cases/speed.py

@@ -28,6 +28,12 @@ def run_dumping(self):
         return self.api.dumps(self.loaded)
 
     def run(self) -> Any:
+        if self.api._name == "toml":
+            return [
+                "Excluded (heterogeneous arrays not supported)",
+                "Excluded (heterogeneous arrays not supported)",
+            ]
+
         self.run_core = self.run_loading
         out = super().run()
         loading = f"{out:.2f}s ({self.args.iter} iterations)"

toml_bench/sets.py

@@ -10,6 +10,12 @@
     BenchCaseDumpNone,
     BenchCaseLoadNoneLike,
 )
+from .cases.hetero_array import (
+    BenchCaseDumpsHeteroArray,
+    BenchCaseLoadsHeteroArray,
+    BenchCaseDumpsNestedArray,
+    BenchCaseLoadsNestedArray,
+)
 from .cases.keys_order import (
     BenchCaseDumpKeysOrder,
     BenchCaseLoadKeysOrder,
@@ -99,6 +105,53 @@ class BenchSetLoadNoneLike(BenchSetTable):
     case = BenchCaseLoadNoneLike
 
 
+class BenchSetDumpsHeteroArray(BenchSetTable):
+    """How the package dumps a python dictionary with a heterogenous array.
+
+    Literally `<package>.dumps({"v": [1, 1.2, True, "string", [1, 2], {"a": 1, "b": 2}]})`
+    """
+
+    title = "Dumping a heterogenous array"
+    header = "Dumped value or error"
+    api_base = APIBase
+    case = BenchCaseDumpsHeteroArray
+
+
+class BenchSetLoadsHeteroArray(BenchSetTable):
+    """How the package loads a toml string with a heterogenous array.
+
+    Literally `<package>.loads('v = [1, 1.2, True, "string", [1, 2], {"a": 1, "b": 2}]')`
+    """
+
+    title = "Loading a heterogenous array"
+    header = "Loaded as"
+    api_base = APIBase
+    case = BenchCaseLoadsHeteroArray
+
+
+class BenchSetDumpsNestedArray(BenchSetTable):
+    """How the package dumps a python dictionary with a nested array.
+
+    Literally `<package>.dumps({"v": [[1], [1, 2]]})`
+    """
+
+    title = "Dumping a nested array"
+    header = "Dumped value or error"
+    api_base = APIBase
+    case = BenchCaseDumpsNestedArray
+
+
+class BenchSetLoadsNestedArray(BenchSetTable):
+    """How the package loads a toml string with a nested array.
+
+    Literally `<package>.loads('v = [[1], [1, 2]]')`
+    """
+    title = "Loading a nested array"
+    header = "Loaded as"
+    api_base = APIBase
+    case = BenchCaseLoadsNestedArray
+
+
 class BenchSetDumpKeysOrder(BenchSetTable):
     """Whether the package preserves the order of the keys while dumps
     a python dictionary.

toml_bench/suite.py

@@ -6,6 +6,10 @@
     BenchSetDumpValueNone,
     BenchSetDumpListWithNone,
     BenchSetLoadNoneLike,
+    BenchSetDumpsHeteroArray,
+    BenchSetLoadsHeteroArray,
+    BenchSetDumpsNestedArray,
+    BenchSetLoadsNestedArray,
     BenchSetDumpKeysOrder,
     BenchSetLoadKeysOrder,
     BenchSetDumpsUnicode,
@@ -27,6 +31,10 @@ class BenchSuite(BenchSuite):
         BenchSetDumpValueNone,
         BenchSetDumpListWithNone,
         BenchSetLoadNoneLike,
+        BenchSetDumpsHeteroArray,
+        BenchSetLoadsHeteroArray,
+        BenchSetDumpsNestedArray,
+        BenchSetLoadsNestedArray,
         BenchSetDumpKeysOrder,
         BenchSetLoadKeysOrder,
         BenchSetDumpsUnicode,