Skip to content

Commit

Permalink
Merge pull request #754 from ngageoint/xml-error-logging
Browse files Browse the repository at this point in the history 
Xml error logging optimization
  • Loading branch information
@adam-beauchamp
adam-beauchamp authored Oct 4, 2024
2 parents c155714 + d171e8c commit 200dccc
Show file tree
Hide file tree
Showing 20 changed files with 8,976 additions and 39 deletions.
7 changes: 7 additions & 0 deletions scripts/six/nitf-HandleManager.supp
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
nitf::HandleManager_known_leak: suppress documented memory leak in nitf/HandlerManager.hpp:122
Memcheck:Leak
match-leak-kinds: reachable
fun:_Znwm
fun:make_unique<nitf::HandleManager>
}
341 changes: 341 additions & 0 deletions scripts/six/six-bad_xml-profiling.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,341 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"processor\t: 0\n",
"vendor_id\t: GenuineIntel\n",
"cpu family\t: 6\n",
"model\t\t: 165\n",
"model name\t: Intel(R) Core(TM) i7-10850H CPU @ 2.70GHz\n",
"stepping\t: 2\n",
"microcode\t: 0xffffffff\n",
"cpu MHz\t\t: 2712.000\n",
"cache size\t: 256 KB\n",
"physical id\t: 0\n",
"siblings\t: 12\n",
"core id\t\t: 0\n",
"cpu cores\t: 6\n",
"apicid\t\t: 0\n",
"initial apicid\t: 0\n",
"fpu\t\t: yes\n",
"fpu_exception\t: yes\n",
"cpuid level\t: 6\n",
"wp\t\t: yes\n",
"flags\t\t: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm pni pclmulqdq dtes64 est tm2 ssse3 fma cx16 xtpr pdcm pcid sse4_1 sse4_2 x2apic movbe popcnt aes xsave osxsave avx f16c rdrand hypervisor lahf_lm abm 3dnowprefetch fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid mpx rdseed adx smap clflushopt ibrs ibpb stibp ssbd\n",
"bogomips\t: 5424.00\n",
"clflush size\t: 64\n",
"cache_alignment\t: 64\n",
"address sizes\t: 36 bits physical, 48 bits virtual\n",
"power management:\n"
]
}
],
"source": [
"%%bash\n",
"# document cpu info\n",
"perl -nle 'print if /^processor/ .. /power management/ or exit 0' < /proc/cpuinfo"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# run commands from repo root\n",
"import os\n",
"topdir = !git rev-parse --show-toplevel\n",
"os.chdir(topdir[0])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"%%bash\n",
"./waf build --target=test_valid_sixsicd,test_valid_sixsidd"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"test_createFakeComplexData: PASSED\n",
"test_read_sicd110_xml: PASSED\n",
"test_read_sicd130_xml: PASSED\n",
"test_read_sicd040_bad_xml runtime (mean/min/max)ms: 34.8972/34.8972/34.8972\n",
"test_read_sicd040_bad_xml: PASSED\n",
"test_read_sicd041_bad_xml runtime (mean/min/max)ms: 9.85791/9.85791/9.85791\n",
"test_read_sicd041_bad_xml: PASSED\n",
"test_read_sicd050_bad_xml runtime (mean/min/max)ms: 26.895/26.895/26.895\n",
"test_read_sicd050_bad_xml: PASSED\n",
"test_read_sicd100_bad_xml runtime (mean/min/max)ms: 32.2583/32.2583/32.2583\n",
"test_read_sicd100_bad_xml: PASSED\n",
"test_read_sicd101_bad_xml runtime (mean/min/max)ms: 27.1155/27.1155/27.1155\n",
"test_read_sicd101_bad_xml: PASSED\n",
"test_read_sicd110_bad_xml runtime (mean/min/max)ms: 9.04346/9.04346/9.04346\n",
"test_read_sicd110_bad_xml: PASSED\n",
"test_read_sicd120_bad_xml runtime (mean/min/max)ms: 29.3579/29.3579/29.3579\n",
"test_read_sicd120_bad_xml: PASSED\n",
"test_read_sicd121_bad_xml runtime (mean/min/max)ms: 29.2188/29.2188/29.2188\n",
"test_read_sicd121_bad_xml: PASSED\n",
"\n",
"real\t0m1.020s\n",
"user\t0m0.234s\n",
"sys\t0m0.063s\n"
]
}
],
"source": [
"%%bash\n",
"# only 1 iterations because this takes so long in the old code\n",
"time SIX_PROFILE_PARSING=1 ./target/six/modules/c++/six.sicd/unittests/test_valid_sixsicd"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"test_createFakeDerivedData: PASSED\n",
"test_createFakeDerivedData_validate: PASSED\n",
"Warning: Unable to find plugin path.\n",
"Specify plugin location by setting environment variable NITF_PLUGIN_PATH, or by building the library from source\n",
"test_read_sidd200_no_LUT: PASSED\n",
"test_read_sidd200_xml: PASSED\n",
"test_read_sidd300_xml: PASSED\n",
"test_read_sidd300_v13_xml: PASSED\n",
"test_read_sidd100_bad_xml runtime (mean/min/max)ms: 221.518/221.518/221.518\n",
"test_read_sidd100_bad_xml: PASSED\n",
"test_read_sidd200_bad_xml runtime (mean/min/max)ms: 315.436/315.436/315.436\n",
"test_read_sidd200_bad_xml: PASSED\n",
"test_read_sidd300_bad_xml runtime (mean/min/max)ms: 314.558/314.558/314.558\n",
"test_read_sidd300_bad_xml: PASSED\n",
"test_read_sidd300_v13_bad_xml runtime (mean/min/max)ms: 318.429/318.429/318.429\n",
"test_read_sidd300_v13_bad_xml: PASSED\n",
"\n",
"real\t0m6.201s\n",
"user\t0m1.016s\n",
"sys\t0m4.547s\n"
]
}
],
"source": [
"%%bash\n",
"# only 1 iterations because this takes so long in the old code\n",
"time SIX_PROFILE_PARSING=1 ./target/six/modules/c++/six.sidd/unittests/test_valid_sixsidd "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"test_createFakeComplexData: PASSED\n",
"test_read_sicd110_xml: PASSED\n",
"test_read_sicd130_xml: PASSED\n",
"test_read_sicd040_bad_xml: exception size (bytes): 1429\n",
"test_read_sicd040_bad_xml: PASSED\n",
"test_read_sicd041_bad_xml: exception size (bytes): 1427\n",
"test_read_sicd041_bad_xml: PASSED\n",
"test_read_sicd050_bad_xml: exception size (bytes): 1447\n",
"test_read_sicd050_bad_xml: PASSED\n",
"test_read_sicd100_bad_xml: exception size (bytes): 1447\n",
"test_read_sicd100_bad_xml: PASSED\n",
"test_read_sicd101_bad_xml: exception size (bytes): 1447\n",
"test_read_sicd101_bad_xml: PASSED\n",
"test_read_sicd110_bad_xml: exception size (bytes): 1445\n",
"test_read_sicd110_bad_xml: PASSED\n",
"test_read_sicd120_bad_xml: exception size (bytes): 1447\n",
"test_read_sicd120_bad_xml: PASSED\n",
"test_read_sicd121_bad_xml: exception size (bytes): 1447\n",
"test_read_sicd121_bad_xml: PASSED\n",
"\n",
"real\t0m1.004s\n",
"user\t0m0.219s\n",
"sys\t0m0.063s\n"
]
}
],
"source": [
"%%bash\n",
"time SIX_PROFILE_STACKSIZE=1 ./target/six/modules/c++/six.sicd/unittests/test_valid_sixsicd"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"test_createFakeDerivedData: PASSED\n",
"test_createFakeDerivedData_validate: PASSED\n",
"Warning: Unable to find plugin path.\n",
"Specify plugin location by setting environment variable NITF_PLUGIN_PATH, or by building the library from source\n",
"test_read_sidd200_no_LUT: PASSED\n",
"test_read_sidd200_xml: PASSED\n",
"test_read_sidd300_xml: PASSED\n",
"test_read_sidd300_v13_xml: PASSED\n",
"test_read_sidd100_bad_xml: exception size (bytes): 1197\n",
"test_read_sidd100_bad_xml: PASSED\n",
"test_read_sidd200_bad_xml: exception size (bytes): 1265\n",
"test_read_sidd200_bad_xml: PASSED\n",
"test_read_sidd300_bad_xml: exception size (bytes): 1291\n",
"test_read_sidd300_bad_xml: PASSED\n",
"test_read_sidd300_v13_bad_xml: exception size (bytes): 1283\n",
"test_read_sidd300_v13_bad_xml: PASSED\n",
"\n",
"real\t0m6.068s\n",
"user\t0m0.953s\n",
"sys\t0m4.500s\n"
]
}
],
"source": [
"%%bash\n",
"time SIX_PROFILE_STACKSIZE=1 ./target/six/modules/c++/six.sidd/unittests/test_valid_sixsidd"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"==11288== Memcheck, a memory error detector\n",
"==11288== Copyright (C) 2002-2017, and GNU GPL'd, by Julian Seward et al.\n",
"==11288== Using Valgrind-3.15.0 and LibVEX; rerun with -h for copyright info\n",
"==11288== Command: ./target/six/modules/c++/six.sidd/unittests/test_valid_sixsidd\n",
"==11288== \n",
"==11288== error calling PR_SET_PTRACER, vgdb might block\n",
"test_createFakeDerivedData: PASSED\n",
"test_createFakeDerivedData_validate: PASSED\n",
"Warning: Unable to find plugin path.\n",
"Specify plugin location by setting environment variable NITF_PLUGIN_PATH, or by building the library from source\n",
"test_read_sidd200_no_LUT: PASSED\n",
"test_read_sidd200_xml: PASSED\n",
"test_read_sidd300_xml: PASSED\n",
"test_read_sidd300_v13_xml: PASSED\n",
"test_read_sidd100_bad_xml: PASSED\n",
"test_read_sidd200_bad_xml: PASSED\n",
"test_read_sidd300_bad_xml: PASSED\n",
"test_read_sidd300_v13_bad_xml: PASSED\n",
"==11288== \n",
"==11288== HEAP SUMMARY:\n",
"==11288== in use at exit: 96 bytes in 1 blocks\n",
"==11288== total heap usage: 4,177,144 allocs, 4,177,143 frees, 1,199,185,878 bytes allocated\n",
"==11288== \n",
"==11288== LEAK SUMMARY:\n",
"==11288== definitely lost: 0 bytes in 0 blocks\n",
"==11288== indirectly lost: 0 bytes in 0 blocks\n",
"==11288== possibly lost: 0 bytes in 0 blocks\n",
"==11288== still reachable: 0 bytes in 0 blocks\n",
"==11288== suppressed: 96 bytes in 1 blocks\n",
"==11288== \n",
"==11288== ERROR SUMMARY: 0 errors from 0 contexts (suppressed: 0 from 0)\n",
"--11288-- \n",
"--11288-- used_suppression: 1 nitf::HandleManager_known_leak: suppress documented memory leak in nitf/HandlerManager.hpp:122 scripts/six/nitf-HandleManager.supp:2 suppressed: 96 bytes in 1 blocks\n",
"==11288== \n",
"==11288== ERROR SUMMARY: 0 errors from 0 contexts (suppressed: 0 from 0)\n"
]
}
],
"source": [
"%%bash\n",
"valgrind --leak-check=full --show-leak-kinds=all -s --suppressions=scripts/six/nitf-HandleManager.supp ./target/six/modules/c++/six.sidd/unittests/test_valid_sixsidd"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"==24586== Memcheck, a memory error detector\n",
"==24586== Copyright (C) 2002-2017, and GNU GPL'd, by Julian Seward et al.\n",
"==24586== Using Valgrind-3.15.0 and LibVEX; rerun with -h for copyright info\n",
"==24586== Command: ./target/six/modules/c++/six.sicd/unittests/test_valid_sixsicd\n",
"==24586== \n",
"==24586== error calling PR_SET_PTRACER, vgdb might block\n",
"test_createFakeComplexData: PASSED\n",
"test_read_sicd110_xml: PASSED\n",
"test_read_sicd130_xml: PASSED\n",
"test_read_sicd040_bad_xml: PASSED\n",
"test_read_sicd041_bad_xml: PASSED\n",
"test_read_sicd050_bad_xml: PASSED\n",
"test_read_sicd100_bad_xml: PASSED\n",
"test_read_sicd101_bad_xml: PASSED\n",
"test_read_sicd110_bad_xml: PASSED\n",
"test_read_sicd120_bad_xml: PASSED\n",
"test_read_sicd121_bad_xml: PASSED\n",
"==24586== \n",
"==24586== HEAP SUMMARY:\n",
"==24586== in use at exit: 0 bytes in 0 blocks\n",
"==24586== total heap usage: 1,171,786 allocs, 1,171,786 frees, 113,915,154 bytes allocated\n",
"==24586== \n",
"==24586== All heap blocks were freed -- no leaks are possible\n",
"==24586== \n",
"==24586== For lists of detected and suppressed errors, rerun with: -s\n",
"==24586== ERROR SUMMARY: 0 errors from 0 contexts (suppressed: 0 from 0)\n"
]
}
],
"source": [
"%%bash\n",
"valgrind --leak-check=full --show-leak-kinds=all ./target/six/modules/c++/six.sicd/unittests/test_valid_sixsicd"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.10"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
10 changes: 10 additions & 0 deletions six/modules/c++/six.sicd/tests/sample_xml/README.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@


*Bad* test files were created with the following script (run from the root of the repository).

```shell
for x in $(fd "\.nitf$" | grep -vi sidd);
do
perl -nE 'if (/(<SI[CD]D.*?>.*<\/SI[CD]D>)/) {$b=$1; $b=~m/.*?:(\d+\.\d+.\d+)/g; $c=$1; $c=~s/\.//g; say $c; $b=~s{(<SICD.*?>)}{$1 . "<fake_tag />"}ex; $fname="six/modules/c++/six.sicd/tests/sample_xml/sicd${c}-bad.xml"; open my $fh, ">", $fname or die $!; print $fh $b; system("xmllint --format ${fname} -o ${fname}")};' $x ;
done
```
Loading

0 comments on commit 200dccc

Please sign in to comment.