Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update feature/perf from master #5993

Closed
wants to merge 489 commits into from
Closed
Changes from 12 commits
Commits
Show all changes
489 commits
Select commit Hold shift + click to select a range
f5248ba
Merge pull request #5792 from xenserver-next/fix-spelling-and-typos-i…
bernhardkaindl Jul 9, 2024
a39d65e
CP-49906: Move scripts/plugins/extauth-hook-AD.py to python3/plugins
ashwin9390 Jul 9, 2024
c8a1b5e
scripts/restore-sr-metadata.py: isort, fix pyright and pylint
bernhardkaindl Jul 9, 2024
deb1731
CP-49906: extauth-hook-AD.py: Fix remaining pylint warnings
ashwin9390 Jul 9, 2024
fb98229
CP-49900: Removed templates folder from python3/
ashwin9390 Jul 9, 2024
0f7a4fd
Merge pull request #5796 from ashwin9390/private/ashwin/CP-49906
psafont Jul 10, 2024
c4f0fd4
Merge pull request #5801 from xenserver-next/xe-reset-networking-cove…
bernhardkaindl Jul 10, 2024
8095204
Merge pull request #5795 from xenserver-next/backup-sr-metadata-2to3-…
bernhardkaindl Jul 10, 2024
4c023d3
XenAPIPlugin.py: Remove the superflous catch & raise of SystemExit
bernhardkaindl Jul 10, 2024
3786289
Merge pull request #5803 from xenserver-next/XenAPIPlugin.py-ditch-su…
psafont Jul 10, 2024
37e615e
CP-50100: Moved backup-sr-metadata.py from scripts/ to python3/libexe…
ashwin9390 Jul 11, 2024
c1ea5b3
CP-50099: Moved restore-sr-metadata.py from scripts/ to python3/libex…
ashwin9390 Jul 11, 2024
8f972d8
CP-49919: mv scripts/extensions/pool_update.precheck to python3/exten…
ashwin9390 Jul 10, 2024
dd7d37a
python3/plugins/test_extauth_hook_AD.py: Assert the current bug
bernhardkaindl Jul 11, 2024
3317717
python3/plugins/extauth-hook-AD.py: Fix logging of run_cmd()
bernhardkaindl Jul 11, 2024
959721e
mypy: Fix and improve the config to make it more usable
bernhardkaindl Jul 12, 2024
160ef6d
Merge pull request #5805 from ashwin9390/private/ashwin/CP-49919
bernhardkaindl Jul 11, 2024
b658808
Merge pull request #5811 from ashwin9390/private/ashwin/CP-50100
bernhardkaindl Jul 11, 2024
df89868
CP-49928: Moved static-vdis from scripts/ to python3/bin directory
ashwin9390 Jul 11, 2024
b1e7710
Merge pull request #5812 from xenserver-next/extauth-ad-fix-byte-logg…
psafont Jul 12, 2024
db484ba
Merge pull request #5818 from ashwin9390/private/ashwin/CP-49928
psafont Jul 12, 2024
91b646d
scripts/generate-iscsi-iqn: Fix inline Python to work in Py3
bernhardkaindl Jul 12, 2024
889b1bf
rm ocaml/idl/ocaml_backend/python: remove obsolete example scripts
bernhardkaindl Jul 12, 2024
418450d
Update scripts/test_mail-alarm.py to test with Python3 and move it
bernhardkaindl Jul 12, 2024
e6f9ff5
CP-49931: Move scripts/xe-reset-networking to python3/bin, fix pyright
ashwin9390 Jul 15, 2024
6b056d5
Merge pull request #5810 from ashwin9390/private/ashwin/CP-49931
bernhardkaindl Jul 15, 2024
a76e404
Merge pull request #5837 from xenserver-next/delete-obsolete-ocaml/id…
bernhardkaindl Jul 15, 2024
f1d09ce
Merge pull request #5800 from ashwin9390/private/ashwin/CP-49900
liulinC Jul 16, 2024
6e37eaf
Merge pull request #5836 from xenserver-next/mv-and-fix-test_mail_ala…
bernhardkaindl Jul 16, 2024
283244c
Merge pull request #5825 from xenserver-next/py3-scripts/generate-isc…
bernhardkaindl Jul 16, 2024
8056f9a
plugins/extauth-hook-AD.py: Fix 'Stray abstractmethod' pytype warning
bernhardkaindl Jul 16, 2024
ed29ab9
plugins/extauth-hook-AD.py: Cleanup obsolete per-method pylint comments
bernhardkaindl Jul 16, 2024
8cab68c
plugins/extauth-hook-AD.py: Modernise testee import to not need a sym…
bernhardkaindl Jul 16, 2024
a0d3be8
plugins/test_extauth_hook_ad.py: Update the patch decorators to 'exta…
bernhardkaindl Jul 16, 2024
4fdea62
Merge pull request #5848 from xenserver-next/extauth-hook-AD-cleanup-…
bernhardkaindl Jul 16, 2024
d1a6ba3
Merge pull request #5813 from xenserver-next/py3-improve-mypy-configu…
bernhardkaindl Jul 16, 2024
31c9ee2
Merge pull request #5845 from xenserver-next/extauth-hook-AD-fix-miss…
liulinC Jul 17, 2024
e10c1ef
Merge pull request #5847 from xenserver-next/extauth-hook-AD-moderniz…
liulinC Jul 17, 2024
3fe07e3
storage-api: __init__.py: Use is_str() to check for string type
bernhardkaindl Jul 16, 2024
d034638
storage/api/{volume,plugin,datapath}.py: Apply isort, darker, import …
bernhardkaindl Jul 16, 2024
3e63602
storage/api/{volume,plugin,datapath}.py: Disable excessive pylint war…
bernhardkaindl Jul 16, 2024
f74d7c1
storage/api/{volume,plugin,datapath}.py: Apply automatic conversion t…
bernhardkaindl Jul 16, 2024
d83218f
storage-api: Add pytest for datapath.Datapath_server_dispatcher()
bernhardkaindl Jul 16, 2024
c23f314
storage-api: Use Datapath_test.attach to test Datapath_server_dispatc…
bernhardkaindl Jul 16, 2024
8544efe
extauth-hook-AD.py: Delete PBIS code (PBIS was removed from XS8)
bernhardkaindl Jul 16, 2024
9c20863
Merge pull request #5828 from xenserver-next/extauth-hook-AD-delete-o…
liulinC Jul 17, 2024
9e9f1be
Merge branch 'master' into private/stephenche/sync_with_master
stephenchengCloud Jul 18, 2024
ae5f7b0
pytest: Run xenopsd tests, rm pytest-cov (obsolete), update pyright
bernhardkaindl Jul 18, 2024
97e117d
Python3 CI: fix import error
stephenchengCloud Jul 18, 2024
d7d376c
Merge pull request #5860 from stephenchengCloud/private/stephenche/sy…
liulinC Jul 19, 2024
e21cc3c
Merge pull request #5854 from xenserver-next/storage-api-cleanup-stri…
psafont Jul 19, 2024
7656999
Update test storage.dummyv5/plugin.py: Base the test on the new v5 API
bernhardkaindl Jul 22, 2024
a26ce26
Remove ocaml/xapi-storage-script/test/volume/org.xen.xapi.storage.dummy
bernhardkaindl Jul 22, 2024
910289f
Remove ocaml/xapi-storage/python/xapi/storage/api/{volume,plugin,*dat…
bernhardkaindl Jul 22, 2024
2f329a4
Merge pull request #5873 from xenserver-next/remove-legacy-SMAPIv3-API
bernhardkaindl Jul 23, 2024
ffacc13
Merge pull request #5867 from xenserver-next/pytest-update-paths-and-…
liulinC Jul 23, 2024
724a1c0
CP-49148: perfmon.service is not loaded
stephenchengCloud Jul 24, 2024
7a78d31
Merge pull request #5879 from stephenchengCloud/private/stephenche/CP…
bernhardkaindl Jul 24, 2024
f0272f3
Merge branch 'master' into sync_with_master
stephenchengCloud Jul 31, 2024
67b28e0
Merge pull request #5894 from stephenchengCloud/sync_with_master
gangj Jul 31, 2024
04377fe
CP-49148: Fix ambiguous python shebang for XS9
stephenchengCloud Jul 30, 2024
e491d05
Merge pull request #5893 from stephenchengCloud/private/stephenche/CP…
liulinC Jul 31, 2024
2facf21
CP-49148: fix pylint warning
stephenchengCloud Jul 31, 2024
c398741
Merge pull request #5897 from stephenchengCloud/private/stephenche/CP…
liulinC Jul 31, 2024
81f1240
CP-49148: Remove unused xc.py
stephenchengCloud Jul 31, 2024
946ca37
CP-49148: Convert rrdd-example.py to python3
stephenchengCloud Jul 31, 2024
355b55a
Merge pull request #5900 from stephenchengCloud/private/stephenche/rr…
psafont Jul 31, 2024
21174bb
Merge pull request #5899 from stephenchengCloud/private/stephenche/CP…
liulinC Aug 1, 2024
8899586
CP-46112: Remove python2 compatible code from XenAPI
stephenchengCloud Aug 1, 2024
0e5ff89
CP-46112: Relocate XenAPI to python3 directory
stephenchengCloud Aug 1, 2024
17da740
Fix pylint warnings
stephenchengCloud Aug 2, 2024
eddf404
Delete unused sed command in python3 Makefile
stephenchengCloud Aug 2, 2024
fcd2edf
Remove `universal=1` from setup.cfg to only support python3
stephenchengCloud Aug 2, 2024
02a7154
xenopsd: Add pytest for common.VIF.get_locking_mode()
bernhardkaindl Aug 7, 2024
2e883b7
Removed shebang from XenAPI.py and XenAPIPlugin.py as they are librar…
stephenchengCloud Aug 3, 2024
c8d9d13
Remove python2 related CI
stephenchengCloud Aug 3, 2024
e7c9da8
xenopsd: Fix warnings: remove inner function, use isinstance
bernhardkaindl Aug 7, 2024
f4c808e
Remove duplicated line.
stephenchengCloud Aug 4, 2024
0360ee8
Merge pull request #5903 from stephenchengCloud/private/stephenche/dr…
liulinC Aug 5, 2024
7b52705
Merge pull request #5908 from stephenchengCloud/private/stephenche/re…
liulinC Aug 5, 2024
fd913f6
xenopsd: remove the orphaned qemu-vif-script (qemu backend was removed)
bernhardkaindl Aug 8, 2024
8e6d4bf
xenopsd: remove the orphaned common.Interface.online() method
bernhardkaindl Aug 7, 2024
7e35677
xenopsd: as Interface.online() is removed, remove get_ethtool() too
bernhardkaindl Aug 7, 2024
806a49c
loop+blkback example: Fix pytype warning: Handle None(not found)
bernhardkaindl Aug 7, 2024
50ae66c
pytype: Enable checking on ocaml dirs, fix pythonpath
bernhardkaindl Aug 8, 2024
dda5205
Use templates to generate `Types.java`
danilo-delbusso Apr 23, 2024
b175c0d
Use templates to generate Java classes
danilo-delbusso Apr 24, 2024
b05efa9
CP-38343: xenopsd: GC and memory RRD stats
edwintorok Oct 11, 2021
8473c35
CP-38343: use sscanf to parse /proc
edwintorok Aug 5, 2024
5446171
CA-396743: log non managed devices in PIF.scan
edwintorok Aug 6, 2024
5acf001
CA-396743: make Network.managed reflect PIF.managed
edwintorok Aug 6, 2024
2c7bc39
CA-396743: forbid setting NBD purpose on unmanaged networks
edwintorok Aug 6, 2024
1adffbd
CA-396743: fix bridge name for unmanaged devices
edwintorok Aug 6, 2024
be2ad05
Merge pull request #5921 from xenserver-next/fix-pytype-remove-unused…
bernhardkaindl Aug 8, 2024
27a3732
Merge pull request #5923 from xenserver-next/xenopsd-remove-orphaned-…
edwintorok Aug 8, 2024
ab53bc3
Merge branch 'master' into private/linl/dev
liulinC Aug 9, 2024
d3b3c74
CP-49148: Clean py2 compatible code
liulinC Aug 9, 2024
eb021b3
Merge pull request #5924 from liulinC/private/linl/dev
gangj Aug 9, 2024
2f9e969
Merge pull request #5925 from liulinC/private/linl/py3
liulinC Aug 9, 2024
7d6967a
Extend Java deserialization support for xen-api dates
danilo-delbusso Jul 10, 2024
88f4297
Merge remote-tracking branch 'gh/master' into private/linl/dev
liulinC Aug 14, 2024
c44a026
Add missing typing stubs for xcp.cmd and xcp.compat
bernhardkaindl Aug 14, 2024
5e40c09
rrdd: Test the changed rrdd.API.update() method
bernhardkaindl Aug 14, 2024
73cf972
Merge pull request #5938 from xenserver-next/py3_pytest_rrdd_api_data…
edwintorok Aug 15, 2024
a5d13be
Merge pull request #5937 from liulinC/private/linl/dev
liulinC Aug 15, 2024
bdf5268
CP-49148: More clean python2 code
liulinC Aug 20, 2024
09603bd
Merge pull request #5944 from liulinC/private/linl/dev
psafont Aug 20, 2024
9601bb1
Update record_util tests to the current state
last-genius Aug 21, 2024
42f84e6
IH-689: record_util tests should accept a more lenient new of_string
last-genius Aug 22, 2024
bc0912c
IH-689: Make old record_util exception behaviour consistent with the …
last-genius Aug 22, 2024
cd7f9ed
IH-689: Autogenerate a new record_utils.ml
last-genius Aug 21, 2024
09c73a4
IH-689: Include auto-generated record_util
last-genius Aug 23, 2024
8a4029d
IH-689: record_util.ml - remove manually-specified vm_power_state fun…
last-genius Aug 23, 2024
94dcead
IH-689: record_util.ml - move to autogenerated vdi_operations
last-genius Aug 22, 2024
971119a
IH-689: record_util.ml - move to autogenerated sdn_controller_protocol
last-genius Aug 22, 2024
b4e2072
IH-689: record_util.ml - move to autogenerated vusb_operations
last-genius Aug 22, 2024
7fe6856
IH-689: record_util.ml - move to autogenerated vbd_operations
last-genius Aug 22, 2024
9bbfef1
IH-689: record_util.ml - move to autogenerated pool_operations
last-genius Aug 22, 2024
66c4c10
IH-689: record_util.ml - move to autogenerated sriov_configuration_mode
last-genius Aug 22, 2024
27f827b
IH-689: record_util.ml - move to autogenerated update_guidances
last-genius Aug 23, 2024
687d757
IH-689: record_util.ml - move to autogenerated vif_operations
last-genius Aug 23, 2024
2859b38
IH-689: record_util.ml - move to autogenerated repo_origin
last-genius Aug 23, 2024
ac679ee
IH-689: record_util.ml - move to autogenerated vtpm_operations
last-genius Aug 23, 2024
1e1aa6a
IH-689: record_util.ml - remove manually-specified vbd_mode and vbd_type
last-genius Aug 23, 2024
7770369
IH-689: record_util.ml - move to autogenerated cls conversion functions
last-genius Aug 22, 2024
bc1504d
IH-689: record_util.ml - move to autogenerated vmss_{type,frequency}
last-genius Aug 23, 2024
57683a7
IH-689: record_util.ml - move to autogenerated vif_locking_mode
last-genius Aug 23, 2024
a229d8b
IH-689: record_util.ml - remove manually-specified
last-genius Aug 23, 2024
f2f359a
IH-689: record_util.ml - remove on_{softreboot,normal_exit,crash,vdi_…
last-genius Aug 23, 2024
1e851c8
IH-689: record_util.ml - move to autogenerated vm_uefi_mode
last-genius Aug 23, 2024
d5f72c3
IH-689: Remove trivial manually-specified record_util conversion func…
last-genius Aug 23, 2024
aa4c447
IH-689: Justify intentional shadowing in record_util.ml
last-genius Aug 23, 2024
cf8ba8b
Add feature flag to block starting VMs
robhoes Jun 20, 2024
8f4c193
Add feature flag to block starting VM appliances
robhoes Jul 12, 2024
7143ce8
Update datamodel lifecycle
robhoes Aug 28, 2024
f2d3a6a
http-lib: log reason that causes lack of response
psafont Aug 28, 2024
6b1c3e5
Merge pull request #5954 from psafont/private/paus/unknown-error
psafont Aug 28, 2024
905a485
Merge pull request #5952 from robhoes/startflag
robhoes Aug 28, 2024
fe21d69
CP-51278: define import_activate datapath operation
MarkSymsCtx Aug 27, 2024
e5088b6
quicktest: disable open 1024 fds on startup for now
edwintorok Aug 29, 2024
bb0411c
Fixup link.
gangj Aug 30, 2024
7d98cc5
Update VM failover planning document.
gangj Aug 30, 2024
b60a081
xe autocompletion: Only show required/optional prefixes when paramete…
last-genius Sep 2, 2024
a4ecaac
Merge pull request #5956 from MarkSymsCtx/CP-51278
robhoes Sep 2, 2024
9156a03
Merge pull request #5962 from gangj/private/gangj/doc_update_host-eva…
gangj Sep 3, 2024
0ac2b0b
Merge pull request #5965 from last-genius/private/asultanov/fix-autoc…
last-genius Sep 3, 2024
4594bf2
Introduce mli for xapi_clustering
Vincent-lau Aug 30, 2024
5d15761
Make Daemon.enabled as an Atomic.t
Vincent-lau Aug 30, 2024
801b087
CA-398438: Signal exit to the watcher thread
Vincent-lau Aug 30, 2024
53a8b49
Remove the condition check for Daemon.enabled
Vincent-lau Aug 30, 2024
24b4626
Add more debug messages for watcher creation
Vincent-lau Sep 3, 2024
911bc2d
xe autocompletion: Exclude previously entered parameters before deciding
last-genius Sep 3, 2024
b1fbddc
Merge pull request #5967 from last-genius/private/asultanov/autocompl…
last-genius Sep 4, 2024
59664cc
Merge pull request #5896 from xapi-project/feature/py3
liulinC Sep 4, 2024
2ba0399
fix(CI): feature/py3 has been merged, refer to master now
edwintorok Sep 4, 2024
b935761
fix(WLS): disable non-root unit test
edwintorok Sep 4, 2024
d9534cf
Update the docs for Volume.compose
MarkSymsCtx Sep 4, 2024
fe6fb4c
Merge pull request #5971 from edwintorok/nofeaturepy3
edwintorok Sep 4, 2024
b01cdbe
CP-51042: Introduce new SR.scan2 for SMAPI{v1,v2,v3}
Vincent-lau Aug 29, 2024
23c4a0c
Replace Xapi_sr.scan with Xapi_sr.scan2
Vincent-lau Aug 29, 2024
2433d7b
Merge pull request #5951 from last-genius/private/asultanov/autogener…
last-genius Sep 5, 2024
1c37219
CP-50422: Destroy authentication cache in disable_external_auth
contificate Sep 4, 2024
b84e057
Merge pull request #5959 from Vincent-lau/private/shul2/sr-scan2
Vincent-lau Sep 5, 2024
f258189
CP-32625: xenops-cli - replace handwritten JSON prettifier with yojson
last-genius Sep 3, 2024
41bde42
Minor doc corrections.
kc284 Sep 5, 2024
5927541
Removed entries that don't correspond to API messages. Removed obsole…
kc284 May 7, 2024
a7c6f92
C SDK: curl flags are not needed since the SDK does not depend on curl.
kc284 Sep 5, 2024
0c1a5d1
IH-666: Report guest AD domain name and host name in the API
last-genius Sep 3, 2024
cdbea98
Merge pull request #5970 from last-genius/private/asultanov/IH-666
last-genius Sep 9, 2024
98d4689
Merge pull request #5963 from Vincent-lau/private/shul2/chealth-delay
robhoes Sep 9, 2024
e9da493
Merge pull request #5973 from edwintorok/wsl2fix
robhoes Sep 9, 2024
9e69194
Merge pull request #5974 from MarkSymsCtx/update_docs
robhoes Sep 9, 2024
e40fe60
CP-47617: Expose backwards compat info to update packaging tooling
Jun 25, 2024
397de7b
CP-46933: Expose XAPI API version in the output of HTTP API /updates
Jul 4, 2024
cf27ff9
Expand Go deserialization support for xen-api dates
danilo-delbusso Jul 17, 2024
8f4c112
Expand C# deserialization support for xen-api dates
danilo-delbusso Jul 17, 2024
dcfc0e4
Expand C deserialization support for xen-api dates
danilo-delbusso Jul 17, 2024
820fb80
[maintenance]: mark data only dirs as such
edwintorok Sep 3, 2024
710687f
[maintenance] disable preprocessor for modules that do not need them
edwintorok Sep 3, 2024
beab7a5
[maintenance] only copy test_data when running tests
edwintorok Sep 3, 2024
b5ef5a6
[maintenance]: reduce run count for test_timer
edwintorok Sep 3, 2024
ec5e39f
[maintenance]: speed up device_number_test
edwintorok Sep 3, 2024
9269092
[maintenance]: reduce iteration count for unixext_test
edwintorok Sep 3, 2024
717a041
[maintenance]: speed up vhd tests
edwintorok Sep 3, 2024
a1f994a
[maintenance]: reduce sleeps in concur-rpc-test.sh
edwintorok Sep 3, 2024
30f24ac
[maintenance]: vhd_format_lwt_test: speed up by using Cstruct.compare
edwintorok Sep 3, 2024
cb1d3a2
Merge pull request #5969 from edwintorok/private/edvint/buildopt
edwintorok Sep 9, 2024
08eaef6
Split generation of Types.java into separate functions
danilo-delbusso Apr 25, 2024
268158a
Split generation of classes into separate functions
danilo-delbusso Apr 25, 2024
a899ee6
Add misc formatting adjustments
danilo-delbusso Apr 25, 2024
101d1a8
Merge pull request #5917 from edwintorok/private/edvint/ibft
robhoes Sep 10, 2024
e72b173
Do not ignore output of executing hooks
Vincent-lau Sep 5, 2024
ca7a981
New hook to execute before xapi shuts down
Vincent-lau Sep 5, 2024
c13b8da
CA-397788: Execute pre shutdown hook for xapi
Vincent-lau Sep 5, 2024
e84d92d
Add sr to the Sr_unhealthy error constructor
Vincent-lau Sep 9, 2024
b9e9369
Add more description on sr health
Vincent-lau Aug 21, 2024
0b22e4f
Refactor sr_scan2_impl
Vincent-lau Sep 9, 2024
9b5d66f
CP-49448: Add handling logic for SR health state
Vincent-lau Sep 9, 2024
88a47be
Merge pull request #5977 from Vincent-lau/private/shul2/gc-hooks
robhoes Sep 10, 2024
3ba6e6d
CP-51352: Compare before setting a new value in `last_active`
GabrielBuica Sep 3, 2024
67df157
CP-51352: Configurable threshold for updating `last_active`
GabrielBuica Sep 3, 2024
1e924cb
Merge pull request #5522 from edwintorok/private/edvint/stats3
robhoes Sep 10, 2024
0438aee
Merge pull request #5980 from Vincent-lau/private/shul2/sr-health-handle
Vincent-lau Sep 11, 2024
08e1437
Merge pull request #5976 from GabrielBuica/private/dbuica/CP-50723-se…
robhoes Sep 11, 2024
c42ba54
Add Java SDK to SDK actions
danilo-delbusso Apr 25, 2024
7068083
Move `BadAsyncResult` at the bottom of exception list
danilo-delbusso May 9, 2024
b1f6341
Apply changes added in eb34314f0c946fe2ae6dc63cb8079f2b473c8c90
danilo-delbusso Aug 5, 2024
9466a7e
Fix spacing in Types.mustache
danilo-delbusso Sep 10, 2024
eda6239
Merge pull request #5802 from danilo-delbusso/bug/rfc-3339
kc284 Sep 12, 2024
373672b
Merge pull request #5595 from danilo-delbusso/oil/java
kc284 Sep 13, 2024
ce3bf5b
Fix syntax in CustomDateDeserializer.java
danilo-delbusso Sep 13, 2024
347f6e3
xapi-stdext-date: Mark deprecated bindings as such
psafont Jul 11, 2024
7c28696
Clock: Use integers for date's timezone offsets
psafont Jul 10, 2024
11c222d
datamodel_lifecycle: bump to latest version
psafont Sep 12, 2024
8c30988
CA-399187: Allow gencert to be called without groupid
psafont Sep 12, 2024
ae952af
Random small corrections in the SDK and docs. (#5979)
kc284 Sep 13, 2024
f8a5f49
CP-47509: Revisited the setting of response headers to avoid errors w…
kc284 Sep 13, 2024
7ae0080
CP-47509: Revisited the setting of response headers to avoid errors w…
edwintorok Sep 16, 2024
ada0dc7
CA-397599 XSI-1704 implement setter for blocked ops manually
Sep 2, 2024
cd1a2dc
CA-397599 XSI-1704 update schema hash
Sep 12, 2024
72cae2a
CP-50614: Add benchmark for Tracing module
edwintorok Aug 27, 2024
4a1cd13
CP-50614: Tracing: reduce overhead when max spans is hit
edwintorok Aug 28, 2024
dc7c519
CP-50614: tracing: make Tracer.t equivalent to TracerProvider.t
edwintorok Aug 27, 2024
24d286d
CP-50614: Tracing: optimize get_tracer
edwintorok Aug 27, 2024
cf7b52e
CP-50614: Tracing: drop unused function
edwintorok Aug 28, 2024
ad7c2da
CP-50614: Tracing: use a list instead of hashtbl for finished spans
edwintorok Aug 28, 2024
df77068
CP-50614: Tracing: merge attributes more efficiently
edwintorok Aug 27, 2024
82644e8
CP-50614: Tracing: replace locks with atomic operations
edwintorok Sep 2, 2024
1ae2285
CP-50614: Tracing: use a map instead of Span list
edwintorok Sep 2, 2024
3347189
CP-50614: Tracing: be more acceptive of span/trace ids
edwintorok Sep 2, 2024
8b7029a
CP-51483: epoll tests: support QCHECK_LONG_FACTOR
edwintorok Sep 2, 2024
c681c26
[ci]: enable merge queues
edwintorok Sep 16, 2024
aa38831
[ci]: enable merge queues (#5994)
psafont Sep 16, 2024
bcf4cb7
Third-Party DCO Remediation Commit for Gang Ji <[email protected]>
psafont Sep 16, 2024
c6482fd
Third-Party DCO Remediation Commit (#5996)
psafont Sep 16, 2024
f20d127
CP-51483: epoll tests: support QCHECK_LONG_FACTOR (#5992)
edwintorok Sep 16, 2024
85f9271
CA-399187: Allow gencert to be called without groupid (#5987)
psafont Sep 16, 2024
233b96b
Clock: Change implementation of Date to use Ptime.t
psafont Jul 11, 2024
77a7063
idl: add test to ensure datetime parameters are documented correctly
psafont Jul 17, 2024
0f9b1df
datamodel: document how timezone-less datetimes are handled
psafont Jul 17, 2024
96195f7
clock: Add timezone to all parsed dates
psafont Jul 17, 2024
57f657f
Specify OCaml version for ocamlformat's output
last-genius Sep 16, 2024
1d32682
Date: Accept all valid timezones from client, allow sub-second precis…
psafont Sep 16, 2024
8902600
Specify OCaml version for ocamlformat's output (#5997)
last-genius Sep 16, 2024
959ebf6
CP-50615: increase max_spans
edwintorok Sep 16, 2024
9637ebf
opam: declare missing dependencies
psafont Sep 17, 2024
5d6a033
opam: declare missing dependencies (#5998)
psafont Sep 17, 2024
807990b
[ci]: make concurrency group more unique
edwintorok Sep 18, 2024
33f20a9
[ci]: make concurrency group more unique (#5999)
edwintorok Sep 18, 2024
7e9c2a3
Tracing: optimize and reduce overhead (#5958)
edwintorok Sep 18, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
410 changes: 221 additions & 189 deletions ocaml/libs/tracing/tracing.ml

Large diffs are not rendered by default.

41 changes: 32 additions & 9 deletions ocaml/libs/tracing/tracing.mli
Original file line number Diff line number Diff line change
@@ -54,18 +54,40 @@ module SpanEvent : sig
type t = {name: string; time: float; attributes: string Attributes.t}
end

module SpanContext : sig
module Span_id : sig
type t

val context : string -> string -> t
val make : unit -> t

val compare : t -> t -> int

val of_string : string -> t

val to_string : t -> string
end

module Trace_id : sig
type t

val make : unit -> t

val compare : t -> t -> int

val of_string : string -> t

val to_string : t -> string
end

module SpanContext : sig
type t

val to_traceparent : t -> string

val of_traceparent : string -> t option

val trace_id_of_span_context : t -> string
val trace_id_of_span_context : t -> Trace_id.t

val span_id_of_span_context : t -> string
val span_id_of_span_context : t -> Span_id.t
end

module Span : sig
@@ -98,17 +120,20 @@ module Span : sig
val get_attributes : t -> (string * string) list
end

module TraceMap : module type of Map.Make (Trace_id)

module SpanMap : module type of Map.Make (Span_id)

module Spans : sig
val set_max_spans : int -> unit

val set_max_traces : int -> unit

val span_count : unit -> int

val since : unit -> (string, Span.t list) Hashtbl.t
val since : unit -> Span.t list * int

val dump :
unit -> (string, Span.t list) Hashtbl.t * (string, Span.t list) Hashtbl.t
val dump : unit -> Span.t SpanMap.t TraceMap.t * (Span.t list * int)
end

module Tracer : sig
@@ -140,8 +165,6 @@ module Tracer : sig
val finish :
?error:exn * string -> Span.t option -> (Span.t option, exn) result

val span_is_finished : Span.t option -> bool

val span_hashtbl_is_empty : unit -> bool

val finished_span_hashtbl_is_empty : unit -> bool
33 changes: 19 additions & 14 deletions ocaml/libs/tracing/tracing_export.ml
Original file line number Diff line number Diff line change
@@ -83,13 +83,24 @@ module Content = struct
)
in
{
id= s |> Span.get_context |> SpanContext.span_id_of_span_context
; traceId= s |> Span.get_context |> SpanContext.trace_id_of_span_context
id=
s
|> Span.get_context
|> SpanContext.span_id_of_span_context
|> Span_id.to_string
; traceId=
s
|> Span.get_context
|> SpanContext.trace_id_of_span_context
|> Trace_id.to_string
; parentId=
s
|> Span.get_parent
|> Option.map (fun x ->
x |> Span.get_context |> SpanContext.span_id_of_span_context
x
|> Span.get_context
|> SpanContext.span_id_of_span_context
|> Span_id.to_string
)
; name= s |> Span.get_name
; timestamp= int_of_float (Span.get_begin_time s *. 1000000.)
@@ -248,19 +259,15 @@ module Destination = struct
| Bugtool ->
(file_export, "Tracing.File.export")
in
let all_spans =
Hashtbl.fold (fun _ spans acc -> spans @ acc) traces []
in
let all_spans, count = traces in
let attributes =
[
("export.span.count", all_spans |> List.length |> string_of_int)
; ("export.endpoint", endpoint_to_string endpoint)
; ( "xs.tracing.spans_table.count"
, Spans.span_count () |> string_of_int
)
; ( "xs.tracing.finished_spans_table.count"
, traces |> Hashtbl.length |> string_of_int
)
; ("xs.tracing.finished_spans_table.count", string_of_int count)
]
in
let@ _ = with_tracing ~parent ~attributes ~name in
@@ -273,17 +280,15 @@ module Destination = struct
debug "Tracing: unable to export span : %s" (Printexc.to_string exn)

let flush_spans () =
let span_list = Spans.since () in
let attributes =
[("export.traces.count", Hashtbl.length span_list |> string_of_int)]
in
let ((_span_list, span_count) as span_info) = Spans.since () in
let attributes = [("export.traces.count", string_of_int span_count)] in
let@ parent =
with_tracing ~parent:None ~attributes ~name:"Tracing.flush_spans"
in
TracerProvider.get_tracer_providers ()
|> List.filter TracerProvider.get_enabled
|> List.concat_map TracerProvider.get_endpoints
|> List.iter (export_to_endpoint parent span_list)
|> List.iter (export_to_endpoint parent span_info)

let delay = Delay.make ()

153 changes: 153 additions & 0 deletions ocaml/tests/bench/bechamel_simple_cli.ml
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
open Bechamel
open Toolkit

(* Bechamel doesn't provide before/after hooks, just allocate/free, but those are done outside the place where
Bechamel checks for GC live words stabilization.
*)
let before_after ~before ~after ~get ~label ~unit =
let shared_state = Atomic.make None and called = Atomic.make 0 in
let module BeforeAfter = struct
type witness = int Atomic.t

let make () = Atomic.make 0

let load t = Atomic.set t 0

let unload _ = ()

let label _ = label

let unit _ = unit

let get _ =
(*
We get added to the instances both at the beginning and the end, so we get called 4 times:

get () - 0: None -> state := before ()
time ()
get () - 1

benchmark_loop ()

get () - 2
time ()
get () - 3, after state, state := None

We want the time measurement to be as close to the benchmark loop as possible,
so we perform operations only on call 1 and 4
*)
let phase = Atomic.fetch_and_add called 1 mod 4 in
let old = Atomic.get shared_state in
match (old, phase) with
| None, 0 ->
before () |> Option.some |> Atomic.set shared_state ;
0.
| Some state, (1 | 2) ->
get state
| Some state, 3 ->
let r = get state in
Atomic.set shared_state None ;
after state ;
r
| None, _ ->
assert false
| Some _, _ ->
assert false
end in
let measure = Measure.register (module BeforeAfter) in
Measure.instance (module BeforeAfter) measure

let skip_label = "workload"

let thread_workload ~before ~run ~after =
let before () =
let state = before ()
and stop = Atomic.make false
and loops = Atomic.make 0 in
let thread_worker () =
while not (Atomic.get stop) do
Sys.opaque_identity (run state : unit) ;
Atomic.incr loops
done
in
let t = Thread.create thread_worker () in
(state, stop, loops, t)
and after (state, stop, _loops, worker) =
Atomic.set stop true ; Thread.join worker ; after state
and get (_, _, loops, _) = Atomic.fetch_and_add loops 1 |> float_of_int in
before_after ~before ~after ~get ~label:skip_label ~unit:"loops"

(* based on bechamel example code *)

(* For very short benchmarks ensure that they get to run long enough to switch threads
a few times.
Bechamel has both an iteration count and time limit, so this won't be a problem for slower benchmarks.
*)
let limit = 10_000_000

let benchmark ~instances tests =
let cfg = Benchmark.cfg ~limit ~quota:(Time.second 10.0) () in
Benchmark.all cfg instances tests

let analyze ~instances raw_results =
let ols ~bootstrap =
Analyze.ols ~bootstrap ~r_square:true ~predictors:[|Measure.run|]
in
let results =
List.map
(fun instance ->
let f bootstrap = Analyze.all (ols ~bootstrap) instance raw_results in
try f 3000 with _ -> f 0
)
instances
in
(Analyze.merge (ols ~bootstrap:3000) instances results, raw_results)

open Notty_unix

let img (window, results) =
Bechamel_notty.Multiple.image_of_ols_results ~rect:window
~predictor:Measure.run results
|> eol

let not_workload measure = not (Measure.label measure = skip_label)

let run_and_print instances tests =
let results, _ =
tests
|> benchmark ~instances
|> analyze ~instances:(List.filter not_workload instances)
in
let window =
match winsize Unix.stdout with
| Some (w, h) ->
{Bechamel_notty.w; h}
| None ->
{Bechamel_notty.w= 80; h= 1}
in
img (window, results) |> eol |> output_image ;
results
|> Hashtbl.iter @@ fun label results ->
if label = Measure.label Instance.monotonic_clock then
let units = Bechamel_notty.Unit.unit_of_label label in
results
|> Hashtbl.iter @@ fun name ols ->
Format.printf "%s (%s):@, %a@." name units Analyze.OLS.pp ols

let cli ?(always = []) ?(workloads = []) tests =
let instances =
always
@ Instance.[monotonic_clock; minor_allocated; major_allocated]
@ always
in
List.iter (fun i -> Bechamel_notty.Unit.add i (Measure.unit i)) instances ;
Format.printf "@,Running benchmarks (no workloads)@." ;
run_and_print instances tests ;

if workloads <> [] then (
Format.printf "@,Running benchmarks (workloads)@." ;
List.iter (fun i -> Bechamel_notty.Unit.add i (Measure.unit i)) workloads ;
(* workloads come first, so that we unpause them in time *)
let instances = workloads @ instances @ workloads in
run_and_print instances tests
)
87 changes: 87 additions & 0 deletions ocaml/tests/bench/bench_tracing.ml
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
open Bechamel

let ( let@ ) f x = f x

(* TODO: before *)

let trace_test_inner span =
let@ span =
Tracing.with_child_trace
~attributes:[("foo", "testing")]
span ~name:__FUNCTION__
in
let@ _ =
Tracing.with_child_trace ~attributes:[("bar", "val")] span ~name:"test"
in
Sys.opaque_identity ignore ()

let trace_test_span _ = Tracing.with_tracing ~name:__FUNCTION__ trace_test_inner

let trace_test_off _ = trace_test_inner None

let uuid = "TEST"

let export_thread =
(* need to ensure this isn't running outside the benchmarked section,
or bechamel might fail with 'Failed to stabilize GC'
*)
let after _ = Tracing_export.flush_and_exit () in
Bechamel_simple_cli.thread_workload ~before:Tracing_export.main ~after
~run:ignore

let workload1 =
Bechamel_simple_cli.thread_workload ~before:ignore ~after:ignore
~run:trace_test_span

let create_gc_work =
let a = Array.make 1_000 "" in
fun () ->
(* create work for the GC by continously creating a lot of short lived strings *)
Sys.opaque_identity (Array.iteri (fun i _ -> a.(i) <- String.make 2 'x') a)

let workload2 =
Bechamel_simple_cli.thread_workload ~before:ignore ~after:ignore
~run:create_gc_work

let workloads = [workload1; workload2]

let allocate () =
Tracing.TracerProvider.create ~enabled:true ~attributes:[] ~endpoints:[]
~name_label:__MODULE__ ~uuid ;
Tracing_export.main ()

let free t =
Tracing.TracerProvider.destroy ~uuid ;
Tracing_export.flush_and_exit () ;
Thread.join t

let test_tracing_on ?(overflow = false) ~name f =
let allocate () =
if overflow then (
Tracing.Spans.set_max_spans 10 ;
Tracing.Spans.set_max_traces 10
) ;
allocate ()
and free t =
if overflow then (
Tracing.Spans.set_max_spans Bechamel_simple_cli.limit ;
Tracing.Spans.set_max_traces Bechamel_simple_cli.limit
) ;
free t
in
Test.make_with_resource ~name ~allocate ~free Test.uniq f

let benchmarks =
Tracing.Spans.set_max_spans Bechamel_simple_cli.limit ;
Tracing.Spans.set_max_traces Bechamel_simple_cli.limit ;
Test.make_grouped ~name:"tracing"
[
Test.make ~name:"overhead(off)" (Staged.stage trace_test_off)
; test_tracing_on ~name:"overhead(on, no span)" (Staged.stage trace_test_off)
; test_tracing_on ~name:"overhead(on, create span)"
(Staged.stage trace_test_span)
; test_tracing_on ~overflow:true ~name:"max span overflow"
(Staged.stage trace_test_span)
]

let () = Bechamel_simple_cli.cli ~always:[export_thread] ~workloads benchmarks
4 changes: 4 additions & 0 deletions ocaml/tests/bench/dune
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
(executable
(name bench_tracing)
(libraries tracing bechamel bechamel-notty notty.unix tracing_export threads.posix fmt notty)
)
Loading