From 003b4aaff638e9a36b4ac4b945516f245b1891ca Mon Sep 17 00:00:00 2001 From: Addison Crump Date: Thu, 8 Aug 2024 15:08:38 +0200 Subject: [PATCH 01/15] archive coverage data alongside corpus archives --- common/experiment_utils.py | 4 ++++ experiment/measurer/measure_manager.py | 10 +++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/common/experiment_utils.py b/common/experiment_utils.py index 3911751ac..15b38d33e 100644 --- a/common/experiment_utils.py +++ b/common/experiment_utils.py @@ -128,6 +128,10 @@ def get_corpus_archive_name(cycle: int) -> str: """Returns a corpus archive name given a cycle.""" return get_cycle_filename('corpus-archive', cycle) + '.tar.gz' +def get_coverage_archive_name(cycle: int) -> str: + """Returns a corpus archive name given a cycle.""" + return get_cycle_filename('coverage-archive', cycle) + '.json' + def get_stats_filename(cycle: int) -> str: """Returns a corpus archive name given a cycle.""" diff --git a/experiment/measurer/measure_manager.py b/experiment/measurer/measure_manager.py index 288148401..a23d1e55f 100644 --- a/experiment/measurer/measure_manager.py +++ b/experiment/measurer/measure_manager.py @@ -614,10 +614,18 @@ def measure_snapshot_coverage( # pylint: disable=too-many-locals # Generate profdata and transform it into json form. snapshot_measurer.generate_coverage_information(cycle) + # Save profdata snapshot. + coverage_archive_dst = experiment_utils.get_coverage_archive_name(cycle) + if filestore_utils.cp(snapshot_measurer.cov_summary_file, + coverage_archive_dst, + expect_zero=False).retcode: + snapshot_logger.warning('Coverage not found for cycle: %d.', cycle) + return None + # Run crashes again, parse stacktraces and generate crash signatures. crashes = snapshot_measurer.process_crashes(cycle) - # Get the coverage of the new corpus units. + # Get the coverage summary of the new corpus units. branches_covered = snapshot_measurer.get_current_coverage() fuzzer_stats_data = snapshot_measurer.get_fuzzer_stats(cycle) snapshot = models.Snapshot(time=this_time, From c96f66c142003bca0fc8c314ef65e278c031800b Mon Sep 17 00:00:00 2001 From: Addison Crump Date: Thu, 8 Aug 2024 15:21:34 +0200 Subject: [PATCH 02/15] format --- common/experiment_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/common/experiment_utils.py b/common/experiment_utils.py index 15b38d33e..30668ab12 100644 --- a/common/experiment_utils.py +++ b/common/experiment_utils.py @@ -128,6 +128,7 @@ def get_corpus_archive_name(cycle: int) -> str: """Returns a corpus archive name given a cycle.""" return get_cycle_filename('corpus-archive', cycle) + '.tar.gz' + def get_coverage_archive_name(cycle: int) -> str: """Returns a corpus archive name given a cycle.""" return get_cycle_filename('coverage-archive', cycle) + '.json' From 90a0a511d749395018ee6384dc95d8bbde463169 Mon Sep 17 00:00:00 2001 From: Addison Crump Date: Thu, 8 Aug 2024 16:55:13 +0200 Subject: [PATCH 03/15] upload to the right place --- experiment/measurer/measure_manager.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/experiment/measurer/measure_manager.py b/experiment/measurer/measure_manager.py index a23d1e55f..789b85a50 100644 --- a/experiment/measurer/measure_manager.py +++ b/experiment/measurer/measure_manager.py @@ -615,7 +615,9 @@ def measure_snapshot_coverage( # pylint: disable=too-many-locals snapshot_measurer.generate_coverage_information(cycle) # Save profdata snapshot. - coverage_archive_dst = experiment_utils.get_coverage_archive_name(cycle) + coverage_archive_dst = os.path.join( + snapshot_measurer.trial_dir, 'corpus', + experiment_utils.get_coverage_archive_name(cycle)) if filestore_utils.cp(snapshot_measurer.cov_summary_file, coverage_archive_dst, expect_zero=False).retcode: From 7821dea46ad17250a8782f2db247317c2de54122 Mon Sep 17 00:00:00 2001 From: Addison Crump Date: Thu, 8 Aug 2024 17:04:25 +0200 Subject: [PATCH 04/15] direction??? --- experiment/measurer/measure_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/experiment/measurer/measure_manager.py b/experiment/measurer/measure_manager.py index 789b85a50..53b84ea16 100644 --- a/experiment/measurer/measure_manager.py +++ b/experiment/measurer/measure_manager.py @@ -615,9 +615,9 @@ def measure_snapshot_coverage( # pylint: disable=too-many-locals snapshot_measurer.generate_coverage_information(cycle) # Save profdata snapshot. - coverage_archive_dst = os.path.join( + coverage_archive_dst = exp_path.filestore(os.path.join( snapshot_measurer.trial_dir, 'corpus', - experiment_utils.get_coverage_archive_name(cycle)) + experiment_utils.get_coverage_archive_name(cycle))) if filestore_utils.cp(snapshot_measurer.cov_summary_file, coverage_archive_dst, expect_zero=False).retcode: From 1cbeb697f956a998890234865863b5df68a45e86 Mon Sep 17 00:00:00 2001 From: Addison Crump Date: Thu, 8 Aug 2024 17:05:13 +0200 Subject: [PATCH 05/15] format --- experiment/measurer/measure_manager.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/experiment/measurer/measure_manager.py b/experiment/measurer/measure_manager.py index 53b84ea16..03cd22944 100644 --- a/experiment/measurer/measure_manager.py +++ b/experiment/measurer/measure_manager.py @@ -615,9 +615,9 @@ def measure_snapshot_coverage( # pylint: disable=too-many-locals snapshot_measurer.generate_coverage_information(cycle) # Save profdata snapshot. - coverage_archive_dst = exp_path.filestore(os.path.join( - snapshot_measurer.trial_dir, 'corpus', - experiment_utils.get_coverage_archive_name(cycle))) + coverage_archive_dst = exp_path.filestore( + os.path.join(snapshot_measurer.trial_dir, 'corpus', + experiment_utils.get_coverage_archive_name(cycle))) if filestore_utils.cp(snapshot_measurer.cov_summary_file, coverage_archive_dst, expect_zero=False).retcode: From 8cdbdb1c940d0381f2d32ffce6bff05c9378dcaf Mon Sep 17 00:00:00 2001 From: Addison Crump Date: Thu, 8 Aug 2024 17:11:43 +0200 Subject: [PATCH 06/15] don't put it in corpus lol --- experiment/measurer/measure_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/experiment/measurer/measure_manager.py b/experiment/measurer/measure_manager.py index 03cd22944..7c16edd2f 100644 --- a/experiment/measurer/measure_manager.py +++ b/experiment/measurer/measure_manager.py @@ -616,7 +616,7 @@ def measure_snapshot_coverage( # pylint: disable=too-many-locals # Save profdata snapshot. coverage_archive_dst = exp_path.filestore( - os.path.join(snapshot_measurer.trial_dir, 'corpus', + os.path.join(snapshot_measurer.trial_dir, 'coverage', experiment_utils.get_coverage_archive_name(cycle))) if filestore_utils.cp(snapshot_measurer.cov_summary_file, coverage_archive_dst, From 1b489249bf0d107f81ee2f8bc342b3697892c6b6 Mon Sep 17 00:00:00 2001 From: Addison Crump Date: Fri, 9 Aug 2024 14:10:08 +0200 Subject: [PATCH 07/15] llvm-cov no-warn --- experiment/measurer/coverage_utils.py | 2 ++ experiment/measurer/test_measure_manager.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/experiment/measurer/coverage_utils.py b/experiment/measurer/coverage_utils.py index 27ab69965..036058bfd 100644 --- a/experiment/measurer/coverage_utils.py +++ b/experiment/measurer/coverage_utils.py @@ -161,6 +161,7 @@ def generate_coverage_report(self): '-n', self.binary_file, f'-instr-profile={self.merged_profdata_file}', + '-no-warn' ] result = new_process.execute(command, expect_zero=False) if result.retcode != 0: @@ -272,6 +273,7 @@ def generate_json_summary(coverage_binary, '-skip-expansions', coverage_binary, f'-instr-profile={profdata_file}', + '-no-warn' ] if summary_only: diff --git a/experiment/measurer/test_measure_manager.py b/experiment/measurer/test_measure_manager.py index 7b6521869..c812239ec 100644 --- a/experiment/measurer/test_measure_manager.py +++ b/experiment/measurer/test_measure_manager.py @@ -156,7 +156,8 @@ def test_generate_summary(mocked_get_coverage_binary, mocked_execute, 'llvm-cov', 'export', '-format=text', '-num-threads=1', '-region-coverage-gt=0', '-skip-expansions', '/work/coverage-binaries/benchmark-a/fuzz-target', - '-instr-profile=/reports/data.profdata' + '-instr-profile=/reports/data.profdata', + '-no-warn' ] assert (len(mocked_execute.call_args_list)) == 1 From 97adb0b259cb312ddfbd76fe8d19567e3eb9a26e Mon Sep 17 00:00:00 2001 From: Addison Crump Date: Fri, 9 Aug 2024 15:03:39 +0200 Subject: [PATCH 08/15] weird presubmit bug fix --- .gitignore | 3 ++ ...se_unique_coverage_heatmap-failed-diff.png | Bin 18471 -> 0 bytes experiment/measurer/coverage_utils.py | 27 +++++------------- experiment/measurer/measure_manager.py | 23 +++++++++++---- experiment/measurer/test_measure_manager.py | 5 ++-- requirements.txt | 2 +- 6 files changed, 31 insertions(+), 29 deletions(-) delete mode 100644 analysis/test_data/pairwise_unique_coverage_heatmap-failed-diff.png diff --git a/.gitignore b/.gitignore index e752b94a2..fcae925db 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,6 @@ docker/generated.mk # Vim backup files. .*.swp + +# Diff files from matplotlib +*-failed-diff.png diff --git a/analysis/test_data/pairwise_unique_coverage_heatmap-failed-diff.png b/analysis/test_data/pairwise_unique_coverage_heatmap-failed-diff.png deleted file mode 100644 index 5dc618e4429bd471b0514411ff94a289000ddef1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 18471 zcmeIacRbbo|37|;ODdvBB)d{5M~Y+|WR^(wo~g_WC3{CUxkOo4B_n(9m32^^?Cq41 z?SzhFobi3WXz2R5-tW)n-{0-$A4li4U(d(5k4Hf_mF1}q96tbq!Kf7#WNyJ=!&n!M}X)?TCTFnFolZCBfR3Qr=X~$2#>#rQCC2s&wR(d0C%$ z>-3`@Yt9F2ro22XwI6w43zfyy7QUpRUZSQywO2zUP`l1xi1LTYrg~tU4_1|Jn1r4!=LGiSyIzy#=(K zRCbQYBhr^Y(3miOiJ6SV<`wwUt~W->GU$jj?x}S&5bmb;5Q*kZJ|NSUwl``yL(&t~ z_ImHSIB)EM2|Z2LqsY=MF&^5v$@N;%kizBtoFRM#sejFdE#HnJo#9#g@6JJQ$Bex9c`H}@A_ zU{oW6!RqLFER;Ly>wTtNstx%meGPMQ6RpvW4VJ~vXDef`o$6sFm#WI|v6PjlDJr@@ z6(iDXU`5x zJ0}SXNtJE+zjiot(^O1-w zMM0dJ|9uA3iiz@J9rLI^MCoe-GuBFM((!Z4vpu{$Ii&ECAsaL<{(UNDbK{O5`?OP~ zYkNmW$9PM`iT(l2@w(1Fs96B;;8lyw|Lp^JKTmsHnrUt4mLh*s@yR;YPh=RwVh$DXLi!P zzmHFjA3h@b&8TLW@X6{DM!__i6Kt-OX1aRz2>Rh9bC}$#4y44FnyC{+v0CB=FWDd9 zYvbgXZ!N(pf*BWRcen2YA7K3<9>G-R`6^L9e5+?K)Qwg9^`|}ND?s|P9Eew&sK{~8 z^KK0F0vpo5!^*99e_DKR#~KU{qQOWml~+}z-+lXWKV5Hg!uG%gYx^%VJ{isRhuPXcPp-DV8+}V`F125o{VE&oxh?ADrN}AJsFh#9OBj@Ob@j z1(`wlp;>yc_0tu(!G>=k;+4!t)tFD5IN_YsH;46QpT@c_f4*&BtZU;%!u33zjk+W4 zCiZN!-?EsK%}7`Gg7bT@Z(f3IH9qhpeiR<~;0nuwKN+L@)05rf%32#Gz+&i6LrOAi z(lZ$^jY`d{K});-P*axh<;$1*2L_(S#~<|lG&aV$kf)o?KrWT7Nhh6^o13}HPTBp@ zCuhW#CYt!?9Q5u(qvpbw|0ongITm|m#axrEJ|Ej2WZ+VrubbWW)S@usxUi5AC%EB9 z$Gs>U{`8b+aa(@%V=WXg3d8Hh+}1;xBSALu79e;U5w0vQE~fUtg2I9d1I94DbJ$Rn z@&!vWGM#P;ih^`%s=TsVWgE%utHqe`_{SwC9z_!?s$8+xisx!yEjSI~2o{Ain>EA% zcFKy8oKc7S_rI+znXS7zI;zz;Z?0y$+q=7Gh3ji;(@L+^_&wVo_y# zwj)=__HT5P-NoZ!a$K8nn@NsqD_<|0g9B1>$R)6wcig>dQtb0b*(*Et5YY{ag5}Cr zYyqX~k3*kHGW+$PLfXz2e`}^?8af4_W zt%HU1!SY7EK_C>6Z_*zHAlr zSb|q^E0xEf`%?-ZOAgWJi8;^D?--(|vu?}NzOmst;XQ>)o^lcZ8^p1{JIixYL;0?r zzCM0!>6N8wSMXcA2%hT+xTpQ+5zO}x2!xTLVR-4>N^E#U#Q0=A9*+lmV+d-XuzZp- zQ*tc(=yuAEY9r%%qzlMz__5X;5ij`|)k9=r--+~jukgbt49K8{$2g%0RRIIvH zB$S51au+iOE3?{X@$K%Bq4V~E{Ca_c$vB0msHmf(V?!u&x8-8KB!5gh8X=BVDG6+&@G1v*$mqCpL^|>yt=uq`&r)9A_+D<6G1~a!%gLznK zK=||!Gg_X*Yh-6_EmHu3GU}~l>R;{a527$KyB+4b5`i_9&LnP--PFeBoP+zA$)Z8# z)y@wHD5rl9M{`wXMq_)g#6jp{L9%pP)mcS9LF!p{b!KOuD?hM4F(GrwBFkb`j9=8s z&W=Cng0S#IG#iw2ou`9cR&^RjM@KWYaxWY@6LW33X|6AKJWM4??C@EX%;xfB_&)9F z@>zdS3O80dz%Fp=#RP{Zcj1hTwNGBsYcfwKVnBYsd$Ay*93v7aMso>l13MY-y8i0f zV|%}S!7(t30`2|A?7X!xFcar^mB?{?ss&b*qrcaz(`e3kE2vqUAzmApP>feG3{DeY}9^YHL6 z{ZW>~xk_;`aAWXC{b}iW_GBE5eYr-DXr*ZR`1mFmDhi%likVy*Z;^f+E(J;$>=P#~ z6b1`TMcR`!*`KK9XI-3#)ovtNfbC-vTc%VWot{QRgP zm51D`@Jo?Rtf`>JG<3s4a7S;`sDXli&?)Y*h-}i-Oy*nOm|$0f^RdYYubR?Q?iwTR zzeyXctgHt(N=iyN(|RIWllW68*ivv9GOrPxCw*42L=yqY~Ih)tjWV|L1=)ysRP;%pkAL}+t~@96{Jjg@b>LkYSIG@UbiX_+ zE4!%n4LhgBAwx@k@ZiDnM>$z1Rc@D;m%HHC^2aaJZ{}Rfqi+mhIr9xXg*rVwJ!`6}PS{1@h?9GSg@w2FjMJsd54@KmpVTX5q^%vP zA@fo*^R*X%9o0oeDawoipJ!)h!TKFz!>>;zPlm8lE;{yrYtB0Ws+gIVmzSGcrT%&F z;uoB{4rOX)LW{sCw-2k)UXYK*1e%o9*77qq&@Zbq%_VWWi(dqN5)wS&eRv*@)zbx@pG@)f%j{CpnPl1cI}7(tDMC- zkj5LC=_0y&=CBHyX|G3sr_pr-3tC zeEufsxJ|CKYp;^S2EDfy>z+=XVWTu5sJk%^4C+riGd4SW?FQTP`1s0-id!nD+z1=R zv}2%_J2`1ms`AA6(}JWOg~`dyZ6BX3U2xj`aC=-(P%w(2L;y70amw>*V6Bi5B%<|| zXo zJCs283T<~lIB1TV^?>GnW~6CMo=+BDbARAx8wsiKm8-Uen-h**E3IuJAA0#rFtbVp z$mege_9tcu40@(8Pz&8;R;n;UOvPejYjFd@>6qsE%~ek6eEqSsZFS}JX`C!{6UMBrlztAokbzJ6uS|gfbl|2T?6+5SGAs< zj7W|12nO|_si{dJ#5Ei7@-kTezwqzX2~%^OTN__`GPTU`tsv066*bX{^>K zR5N>4abC%)=h%GsGTiX4W|=*~?(X;Q-SfB7)7EDEtOic!JRPT|9@K1jL7?os+{TzpAXjD{Ga&vOJ_e~CZLtuCFe8@hI#wirZlc5frXcLFQ-drD0PJ7ix zl}ZEdThc_iL*_TEySRiFu{uMvtu{ohWfhc^q_Tgs_m#bMiX`VpIf@c*coh5vRSdNr_ zTwPr~&mEpI#aQJ@xH5W%E#lK<<*HG_@?<;ZF-LR+c&v4Mg&>5fZSyxeeg_ahuE1r$ zw=e}d0bA%deYnV0w0H-4|3`qk)z&|CA%R`aJe>-=kT%_Z%ncMBP3H8s?3BKflanvc zUk#>jE7o23`ZX^-1bPn~(4zf9I_k0Kr8rQIl$DpS6A5nO>OVQ0&j7=FJ)(_Z6~C~t z0v_zA&nDINKUa7rZj)|mefY-$znrB{#OXhGC>I;dZHj|KAl?x$7XoQWSjf~m{5xq8 zjBZeem^I!S!UGrVY>Z`}m;BH|31;)XMHkM4P$)%vztW(3a(a3ibfnAJFJDsKmsS__ z*2mV?v{}mY^4N{^4Gc1_fGT>;>kKn=!}yeuc@^dZ0K|xzS}pd=38wLP60&vPjxzO|M_tm;iFUcun`$Tl?OSK~ zGtd9Vkqr2F1aC7!L`I+16I!4ruDkebRYdkt55*h!4yof^D~qMzZh zJfi*-v_O(v%%vqI4wbB68)IpDL0fiAFaP=Tp7=G@;&eUitm^F`VJzLy(42?}6W=TCBaqQ+f!uL=}Hb->-!o|YWE4v8wh9L>xH45?=D zmiV&L-3Y~AX1#39iE#iB9vZd|59^l>g(Za9L#^Xkb2WYv$NpO3cP5F-@+>EsPYTU) zvt6*Yv9al7m?`|uheb94T&!L7kam+E;*ImJ3d~V&SAq$B(htnNg8;=A@7bb1T}=~j zW@|p8<>uyYAVG9iUJ%3nqQ<&4m@{u?3hE4ycEL?m5byY+X7G z;cs#ofnY;?*7oW=yn8EO7w1`<>&f!);lnOV??$<5T(UxqUT&YD1aCUEKW#UK9N>#0 zKm6#SxA{(5Q_!)vZ~hD8c{3#F^{ZE}058|x2O8VurlC^1$&m4Jvj-Dz-%6i6W9CZ- zZeqSU;bl;yQz63U3_HH$t~NqqDUT!BR6O60!q=XR3&I-^-3bf9ObXg=Q&pfjFejZ0 zs(PJtqola_lom&FLPC0TXgnclYUCM2NlMaV-Tgq?7x&;%O>kYV+XBMSYTv@a?O$V; z#v>4iyx?F{KEa875I5!q&UN|JH(3*E$J#DowH+#cBxz6A&b^I9Q5P=Fa?XnF~< z5cu(K`iTR6BSj0y+~OF7(vS~E;^UdI3+CaxV0&X%IDoyyNLQ1ad}th(itL$B||`!R>(bb^y`|L`px##Kd%Usa4A^JbL4Ag1ihWUtJNCjc??QSJX}26w9OdSMm}5 zv|G&St-D>p6P5QW4NsP|qvQrrDAetq!}m~&xYD5oK$eeUrogG|8t$m^dtH3={%ogg z5+ZW-vhSluk3yr1bk7lnKwI+q^{EHa9F4QHmdz~@5fPJxC{F^mkAok|YjteJwY9a? z)c}oppfOIB_X0ICHPAQ@bZ+0=hAC{IAA@Uv52MksVI~J$-h@t)n=oqf!~_{Q)a}j` zLHU6dHn15%T2O`H@yQ+v>CJO0d`G$kvNb~nB>^yKeJLb)C1i81n*!>p=HZ7OgCX|x zdFHK*^u9@l_4So0H#fJj%?EooaxtGjzo0d>vf_2nCB((@8=aPGH`6u-*9}whw=D&m z@fu+k7iT?OA8>h@nQ@KAIj7_p=E@gbNep*==;=wEO7>y^0z=xNGqLR(rH{-On8EskT_G>}o>p>S^^7tGPgLKZ3t&J|4WmKXwXMNl%tw8Zf|5cOEDayh}PnL9_e+*>r}^ zpWitNBketrHiMbtmtBAMl;FRXR<~hW3`G6c|js>fpm0xeww&x0oQ%J>@xL*jLX&`j>Iw+tsrU1`jXb zsG#6YRRg%#*48#SIQ+5tX~4vIZFD6ES3pIGzKQvsa>ua{&`SMF7+`Ix4Ub8SdE&Ku zc``FI*(BV*ieUi$D>pa-_CM`&qZ;2quf~kj!-5DpBE6-bU4L;-D;ATw;2QOVB#qUO zPcbk$M(V32YTWf>1V^Ej#?Kj%Oc%tU3%mg?@P7FCw8s*%v<;-PW<}lXr^E#;7TekR zE&f46mWJTqqA2$(BzB?aRr5FKL!@`s!-aE;lTQT37C37 ztxD&ZV-Wdz{2=zC)R`$4O2zUOC>!#I7G}{Uy4i3lAO_*&-%0OAQr0n|R=#oHQ0F;{ ztNzB# zy)h7KJ9-fG{RAR1BR~bU5-|+ljokTzF}n9sY~toxB?y=aXAP?KX3fk?@7jD6&r^{X z`$YYZsDoA?yodC3eqo^r{agHmUBiwiOL$yiq-qo=*@}DOosnEnYNkW6Otznf;>XKZ z3L;F853~&95*m^NTN1$cl9Wrg2iZczm{1Z>M|HF8Z8J4r-EWceZYJLwLe^b|5Q!9Q zlVo3FvES{EakOl&`t2KE8pAqR&4u#r7WoVscM>*4YO`wb)vI>N-TD-j>jjejF3yHXLneWYkyd0l12pcBo8{ZMUQM zLg%O_kljB8xVF>i8}YB7{|Kcmy@94Y`JQQoXCgRo+!4PFqd(!z_I}_8bO(mHJ2=!e zIXe~WP^EoGr1{RBdsYc)FfcP(PrJB=?rzdrOV@F1bVS~wh-W}yVPTMNNQ169NJp|l zkxk%(gJGeS)i${9y}zvg&_DNO?qJGNIP?7c{JZz>sReBTux`Yuk;eTfZb2Z#19^pm z($|Z)l59!Q@%HUoP!NtdLhrYf->;t1NYsS*eJ{R@w9VLetikXVSk^yKC z>)&RgX~({P{rc_OH_+63++6LxV`1^lwwjOCpEemma%&*HfBzm3u5&Mz3qdCs!_>4G zv6WokpfQ;>#Kpy&QH)Aei`P`vi#14q7%vQpQ%f;Z27>7qdXB_j2pbf@<>=lQeR?3n ze8(HrBRGO>X(k(faYPhrTny9?g{MTM@A|q6hyRrqR}|B_6sz9+YlPMjsMMHcva+*f z!&Str;~dT{on)FB$8*~wcGn=9%SI*d(g-NHX?b1GhIy`hxluU47jq5tpl|Z~Kr^r< zp%xVSvIDYFdb1iKA;IvB<`#Hwb?QtuT^#LCxgw%G1r>7y9;1^~Djk!Z_z|P$Tc``{ z^lvBJO!lS+jp{SB2ijz3QyRHKc%b*)905;k76tO0_YPFPwqjnjX+**r;VPg7Qg_YF zy#~BDnD;*aAT0EoR@#}%V>aY-Cae>vZjuahfCTB7!??`Mmsj&(9IP*z9qid zPFCCjo#ht1fCOnE790XJ>lRtcI4&Uzl)Y`iO^Q}uUq6LKGqcB+J-x?L`Y{&!0C+rn zj$ip$JhWianVkidDeonw@r0Mh_zjWA0Qo3QiCUrX^EvLAq0R%6;t}Z+tgJ~%NrnK} z(zLmdz!t8hh72^)k5S2p%;tZre!y5;c%28w)V`N;R@V2d?5iJ)6NJYxI|!$FvZe$; z48MbJ1AO31nrcY+eg~LB)O`^5V?)W2TDgW9PK0CmBevhwhCT!{xm4*;k6w6qI4i=F z&&bZ!w(Xc5x+aDn2%SAq4#Rag4{YD{8X&38?oDwh=QFw(zy1IN-DXDgHu#`w_)b4j z7rqo*IvG_8bQvzY8^5W>%XoB1)AwW>r#$YlsrS*3U{! zpQ%Av3g%4tvv%}Ca?f#C@0QGx*Wf%wU!LRU==iE8@m)O^*i@48bJ)d<=@8lI;AXv_ z^j5yb?XK&eohk|Q1BK3&MHLmUuCC9A9zoevy1%=f?S?W+_WTx$yKcUpLsL^OId}PU z!#+QV0f33oo1GAjES>npb-nyL^s4O$cj>7gux0YGOp3S-@z!cpYIb%uPzLX8#j&0X z*Mi!&rkC{r0=*Pqxi1GbNzs0Dt>#uqI>`!_JK%DSZk=T_;LG6VoD0j=Q8(7C?}wI9rXQMGjl&jZ(pB=8!frzv}7M7Zr9o+veR5wf*y2gJ>%5>^e$<7Y3)Mi zKG~vk_4v2x++>aM-`U(y0MbPHHU5}ciG}V(L)$Jg836i7RF6X*Zh$#g568FGQRLM{;bMxx-_6oSMQ1QFN9 z+JotH|Vq66L$(FPTLH(UPIVk1Bsx{UZMt9ml z_WipZr0+;N+Rek7!vQQchygyq$WO3%qHji9do}rjMWn0fkLvR}5zcj2`nRexMKa<0 z(8diJmEPVzjw$o;JXQ;JcVY8pi*6r2qXY-jphqiS#bNQL#e=C|a2P&1SZ6!jAspGn z;sNBrkq(+4@}RAeNO<&Ac7fP9q>PU$o`RI|=kGQPL5+Ye4Ysa8do`DJ805l*#l?+ z+#I$-V%$h*b&~+6!r^d&tY%qqRC0!b@YXr%AC{^usgQ3}^sz_qmL20H#0U6n@c}7W z%)M5A$g}`_$m?*v*me`fis&Myqq~CZI@Z+nB{3C@qZR-+x~oWt=sG~wGiog~KIsE$ zS8fa9@;m+wAlG{!o-pq!Q{#SUh6+r7yG{UdZK=7D>%p6oH@67_B5R@*YQ>)xIGu2$ z_2x&r0i-&D%$0AG4kDe46*>B5AHOT*zvzJ&sl=P7oUSCUX*6X}`cX1l-QBWGI>9vk=JjE`TCF3W~1x-Zrvh)WToDDVnl+7_WG z2|6^wJ8QRSkQ>UnHJVn^;A{nL`{_kqMl9mErC&$>5zu+@?x8ZDEIqy zes%V;a*~?$N@(Gw>;0l+@xcAc6H>m#GQdKEA%n;kQw+r6UvC^jgahVPj=}nU)#c@? z=nKvucX|h1TF9hst`5vFWTe0D??DKnY=uYX?#FrGy{&W&*vP7*a z0J8+sov7ky%=5&O>;-{`n_>Sz?X(G_ zt(~39vKQ6v?fo+FW4a_6YH4gEeuNj%9c=Dit1L-k5f>Aqm$tUCkyXxQTLEI(;q&vg zq|Mq#tEDziT4~RA957K~>u$z5 zd=M%qTfN)w5(BtnPW13F^4OMtD)bU>yy43t-Rz>$^zF{a;kg=+kRf9sbEdA2j)-_Q z`mY#v%83xtdNPTr;V8H#m@}kyfqsVtbV@8NSl4QH%Jb*XcOJ^@v8)$%bu%?HzY_omBIbydId67IDu8@00bFhmFl8ME z1{?nFwX2K)D=QC&ol%UBEU}gLvsdU5uCY&jmrFn4h`IHNSgkND> zS9}l5Nl=Q1tcS3lRQ8{Udn~TifS6g|HdOTuZBgnQ^_bCWH5$#K{{F}2O)b>j`+F3x zBr1<-Os>n@GVT`S1%A9+k7PFR-VXEOhWcyKt}es5(P}s04_-%bS9jy$`c0}(vLA?k zhh47Q=vM5IT6*3=Qp@Y7mom)&zUn$eY&Rl4S!DoE(Ks_9DTL1-)EGDHtVZ)5uuy88 z2^$ZpmH@Vurfjhv`=bspEsIK8Q>x#@qwb5Lp`ETQ=}&+l1CXW<@|fQ>XLCe59`Z`+ zkbtuDj0q%2j3;h^lTR-`j)spy-Sj&uSPdjPNK7+fVe@wHSAE1cT4VHsZyX;yS!Tirt=mU zfw>gsKF*!X)@+BI=}gaVq*$EFcnYkzKddHW;}o<0vnL%?rJV8w_V0cfD0s0{3pU;h^J*Q9nEF7-DR%u99=eljOW5l?UVJUt0>yJYY zbV=dS!Ck21QzdH6;9Dg@))2ALLCLc_fPFw>Zb3!G@w?{ZVD6>G=^ofD?xtT9Ky=N( zo;894_N)jt#2bd<(+0owTJAvF98D@&6YnGfUVtzJWN`rEfi+lZde8+N#Mru7`}YYB zAZ;m%0Rt7J)Tb1v^JF+ZIA&zmokpezgnAHJ!1gSf7=GZ%vDPw>66)wplV1uPcc7?+ zDvRnq?wmbHL)NeRj!`LnF(uo{qW$CT{8yx3Sp)lt^Xt55pr@t`6v+`Uny&A%3b)hb z?;#Y*j67zE=b&m>Uw>g+rSj!(AJ8i)K=VDnON&lm)CT`;(*rA73h6Rrz+F&Hl35;c ztxj4RpxSU`48`YB?1mW*Iw9}TsBtvT&u6Wza!&x$+ZzLttaa&74MsTr1{#M&QAmAU zaKb))^@^+5)cN<>j=xYEUp z7~n%!wkUtMt8;Sx(}C~oyuhLfPf;La%71XO4CqMFBCBuqV5%ib*!`WqO?Ui>fY?dC z$?ziPvwCrd6l|<;UJanO%dTbvzEB|+L`RfN1!wHK~A54azR ztEIK43{&uHz$kCIg^1KT!QOC#;iwwE3#;~Iw~_J(0J#63vcH_GnEh9q>FLS#*b5Z{ z@9fS2dK`8dX`I4O^&h;z9D9GN|0qUiJdkmj)1WUbd10jTq1RfPaKKJO-6uoRLr&gv z+lJADXP*3mHn&b6pDC7_P%(eFHq4C0DZq(#gfZ z-H=QjAq87|drE?sqyOlHh~ZJPH^9$F?HG^nJhHwqKhH1nF&tQO>HEv*x>sctg_S>W z@i;ZN++7vcvS?4`M0Yu zksTjNQ`jQPR0koV>_9`_z&lJ0KR6g%1hCYU95vgKWDNArVV$N#{%;I)mUTMSxYPZlR~>~7AKv=GwP5Ar}BVo1c37Zi)h)ev{b}qOoD1> zW!s?-!4)Cvz40&WeGIVP&l`PP+rLSunQ}2(=aApkFCZFP;Fn==@X$7mkV$CB+{R3v zMEv^bXwsTzqBl|0)!F%sfB)cMaDpb{3&NjgtFF~k5mWm1tRL;MCq^5Y~ZbA>- z1MhnVMn-xzzs)8Qnw#aS1eO#GstqH|!OSIg5s$wqJi1BnEFAc&53c$^aQ7jvMefcQ z{ITs(PJIt1kNa5^p6!Dv>~P@xpx7D@9yFtoJOgIEcCq5rCA`EpR({$ZN{PvgK!f!3T`&0tdFcD}1ww?K96&1l=z~L=4dM0+?^bU$`#D&T&4kQ-i zD`86mcg8k7op$;NxW^5Gq1d4m7$fJ12MTJb5cmtmdH^Z}FN7FbAX*LL*EBtO_q>It zwd}nDW)W|=fx%bQ*^=#9#6+JP&@5t8V_2(<4SdFvkYLnnL>z5D*Y4u<+^9F1W~JR> zbRrJR*^)ImT)<~xbw_+E%iR|+tXVUS2``uavx%UfK|wE0w>W6V8F#j#ygZ8Emt7n9 z;jJ_S*0>I@;z%PI!Bv+g3ZaT({49z!#JVTF9+bD&+t$tKQ~ugI>x)U>evB`+A6uGA zopgcx0d*6MW;X2dsSL-Cah;oeuttbRau!A$VIE%qBjQV|G;9*nS`O~+Cu+q+BqG=D zK2o^1RjogTf4d#ME&BY|>BVshNTu{Lnh*s8TrRw~bfK#}P9?YT_T~}H#vQg)28u>R zk^nODa!Mq_Lno)xG?8HB_5pw{clt+0-mf_mwL`4gLyD1ryAQFJTqPwUk6-we9{>AT zqu&W(O5aZ#8yhNK8`0KPH8r5s4=M*&k;lAAKiRS(rZps3vR$e~@_BQGH8V8^I<18? z#z8=s4@=g36h3`x^N@|5U5lDKTFZ|@nsR**M-af1F&_Sxh)4sr19UmEDpB+x*$-!D zmjo}}QH^IvJFm?JVeC1_;AK!kIYS0N`Q-nY3f_bJ@7@DV^I`T|j7a)&w|k5MQK}*T z>O9!$UU!%el#<_EG{3*PALv2N1Qj#q-?=oE+p` z|9U2iLNMFeK!1zTlZCjhL!V%7yL$}^8qQ==XX)C)zlg}@%#98Yr`tUn0I1pk@BK{UQ|oMFy|~KhJYh_pS}1GI8_s$TToNOD)I@~8p{nJYd0(T z7y7V#p~%wDfRlhCCXVRt*XTs>YXftKRfL8>nvEC$P+R!4flP-W#$8KPy{HI5`Yq(+6Sd0#z^Nr#3sBL&Q4qk z-I{SJDg#rHE_Es5q|4Mhhq``?39D=6Tpl{+RsTIMw#R+fK7dv-v<^gpUY6?iXjeji zOdoS*&l_tn?eWGQSTir!vKvh^(`79A02bYZg5fmve_Z$@BO^t}+4EH~1nwu$6nqh? zO-43X>GbY4OkhS`SKL7|fbsXyMTn>Gq4C5mCeBqXV?iLEIM&%&xmHiTVI8f0n_2ZG? z-GfP9B|>=0csBKXGbD9Fnr94}h48}zy097pJfNdI?b{yEMD;BXsO<-+cmDr>{y+8v ba5&PQ%-gu*f Date: Fri, 9 Aug 2024 15:37:51 +0200 Subject: [PATCH 09/15] ok, no no-warn --- experiment/measurer/coverage_utils.py | 4 ++-- experiment/measurer/test_measure_manager.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/experiment/measurer/coverage_utils.py b/experiment/measurer/coverage_utils.py index 744e2ed9d..16c0a24a3 100644 --- a/experiment/measurer/coverage_utils.py +++ b/experiment/measurer/coverage_utils.py @@ -150,7 +150,7 @@ def generate_coverage_summary_json(self): def generate_coverage_report(self): """Generates the coverage report and stores in bucket.""" command = [ - 'llvm-cov', 'show', '-no-warn', '-format=html', + 'llvm-cov', 'show', '-format=html', f'-path-equivalence=/,{self.source_files_dir}', f'-output-dir={self.report_dir}', '-Xdemangler', 'c++filt', '-Xdemangler', '-n', self.binary_file, @@ -258,7 +258,7 @@ def generate_json_summary(coverage_binary, """Generates the json summary file from |coverage_binary| and |profdata_file|.""" command = [ - 'llvm-cov', 'export', '-no-warn', '-format=text', '-num-threads=1', + 'llvm-cov', 'export', '-format=text', '-num-threads=1', '-region-coverage-gt=0', '-skip-expansions', coverage_binary, f'-instr-profile={profdata_file}' ] diff --git a/experiment/measurer/test_measure_manager.py b/experiment/measurer/test_measure_manager.py index 033736faf..7b6521869 100644 --- a/experiment/measurer/test_measure_manager.py +++ b/experiment/measurer/test_measure_manager.py @@ -153,7 +153,7 @@ def test_generate_summary(mocked_get_coverage_binary, mocked_execute, snapshot_measurer.generate_summary(CYCLE) expected = [ - 'llvm-cov', 'export', '-no-warn', '-format=text', '-num-threads=1', + 'llvm-cov', 'export', '-format=text', '-num-threads=1', '-region-coverage-gt=0', '-skip-expansions', '/work/coverage-binaries/benchmark-a/fuzz-target', '-instr-profile=/reports/data.profdata' From 3af5fdf3fd1b6c2a86673db914817ada0d367907 Mon Sep 17 00:00:00 2001 From: Addison Crump Date: Fri, 9 Aug 2024 15:45:34 +0200 Subject: [PATCH 10/15] dummy change for experiment runner --- service/gcbrun_experiment.py | 1 - 1 file changed, 1 deletion(-) diff --git a/service/gcbrun_experiment.py b/service/gcbrun_experiment.py index f19ab493d..bbebcf1b9 100644 --- a/service/gcbrun_experiment.py +++ b/service/gcbrun_experiment.py @@ -28,7 +28,6 @@ TRIGGER_COMMAND = '/gcbrun' RUN_EXPERIMENT_COMMAND_STR = f'{TRIGGER_COMMAND} run_experiment.py ' SKIP_COMMAND_STR = f'{TRIGGER_COMMAND} skip' -# A DUMMY COMMENT def get_comments(pull_request_number): From 6723889e2cff94187ffa69b5f13f9b51cb671580 Mon Sep 17 00:00:00 2001 From: Addison Crump Date: Mon, 12 Aug 2024 15:09:43 +0200 Subject: [PATCH 11/15] revert changes in coverage_utils; remove warning lines in save --- experiment/measurer/coverage_utils.py | 25 ++++++++++++++++++------- experiment/measurer/measure_manager.py | 3 ++- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/experiment/measurer/coverage_utils.py b/experiment/measurer/coverage_utils.py index 16c0a24a3..27ab69965 100644 --- a/experiment/measurer/coverage_utils.py +++ b/experiment/measurer/coverage_utils.py @@ -150,11 +150,17 @@ def generate_coverage_summary_json(self): def generate_coverage_report(self): """Generates the coverage report and stores in bucket.""" command = [ - 'llvm-cov', 'show', '-format=html', + 'llvm-cov', + 'show', + '-format=html', f'-path-equivalence=/,{self.source_files_dir}', - f'-output-dir={self.report_dir}', '-Xdemangler', 'c++filt', - '-Xdemangler', '-n', self.binary_file, - f'-instr-profile={self.merged_profdata_file}' + f'-output-dir={self.report_dir}', + '-Xdemangler', + 'c++filt', + '-Xdemangler', + '-n', + self.binary_file, + f'-instr-profile={self.merged_profdata_file}', ] result = new_process.execute(command, expect_zero=False) if result.retcode != 0: @@ -258,9 +264,14 @@ def generate_json_summary(coverage_binary, """Generates the json summary file from |coverage_binary| and |profdata_file|.""" command = [ - 'llvm-cov', 'export', '-format=text', '-num-threads=1', - '-region-coverage-gt=0', '-skip-expansions', coverage_binary, - f'-instr-profile={profdata_file}' + 'llvm-cov', + 'export', + '-format=text', + '-num-threads=1', + '-region-coverage-gt=0', + '-skip-expansions', + coverage_binary, + f'-instr-profile={profdata_file}', ] if summary_only: diff --git a/experiment/measurer/measure_manager.py b/experiment/measurer/measure_manager.py index 82baf0a9d..b3813653d 100644 --- a/experiment/measurer/measure_manager.py +++ b/experiment/measurer/measure_manager.py @@ -626,7 +626,8 @@ def measure_snapshot_coverage( # pylint: disable=too-many-locals with gzip.open(str(coverage_archive_zipped), 'wb') as compressed: with open(snapshot_measurer.cov_summary_file, 'rb') as uncompressed: - compressed.write(uncompressed.read()) + # avoid saving warnings so we can direct import with pandas + compressed.write(uncompressed.readlines()[-1]) coverage_archive_dst = exp_path.filestore(coverage_archive_zipped) if filestore_utils.cp(coverage_archive_zipped, From 9eb92a4a6e36ede1ffc35c0f8cab4a54041c19ee Mon Sep 17 00:00:00 2001 From: Toka Date: Wed, 21 Aug 2024 16:55:02 +0200 Subject: [PATCH 12/15] build --- fuzzers/libafl/builder.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzzers/libafl/builder.Dockerfile b/fuzzers/libafl/builder.Dockerfile index 7fd7462a7..f0136ff46 100644 --- a/fuzzers/libafl/builder.Dockerfile +++ b/fuzzers/libafl/builder.Dockerfile @@ -38,7 +38,7 @@ RUN wget https://gist.githubusercontent.com/tokatoka/26f4ba95991c6e3313999997633 RUN git clone https://github.com/AFLplusplus/LibAFL /libafl # Checkout a current commit -RUN cd /libafl && git pull && git checkout f3433767bea0cc3d7ee3b4c08be138e61d20c468 || true +RUN cd /libafl && git pull && git checkout f856092f3d393056b010fcae3b086769377cba18 || true # Note that due a nightly bug it is currently fixed to a known version on top! # Compile libafl. From e698750a7a95fe5cc7dba260c6accb18a44e2522 Mon Sep 17 00:00:00 2001 From: Dongge Liu Date: Fri, 23 Aug 2024 11:27:46 +1000 Subject: [PATCH 13/15] More informative logs to help debugging --- common/fuzzer_utils.py | 4 ++++ experiment/measurer/coverage_utils.py | 4 +++- experiment/runner.py | 3 ++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/common/fuzzer_utils.py b/common/fuzzer_utils.py index 049ba8ce3..a28acf5db 100644 --- a/common/fuzzer_utils.py +++ b/common/fuzzer_utils.py @@ -83,7 +83,11 @@ def get_fuzz_target_binary(search_directory: str, if os.path.exists(default_fuzz_target_binary): return default_fuzz_target_binary + logs.info('Searching for possible fuzz target in search directory: ' + f'{search_directory}') for root, _, files in os.walk(search_directory): + logs.info(f'Searching for possible fuzz target under subdir {root}: ' + f'{files}') if root == 'uninstrumented': continue for filename in files: diff --git a/experiment/measurer/coverage_utils.py b/experiment/measurer/coverage_utils.py index 27ab69965..9f8e295c9 100644 --- a/experiment/measurer/coverage_utils.py +++ b/experiment/measurer/coverage_utils.py @@ -133,7 +133,9 @@ def merge_profdata_files(self): result = merge_profdata_files(files_to_merge, self.merged_profdata_file) if result.retcode != 0: - logger.error('Profdata files merging failed.') + logger.error( + f'Profdata files merging failed for (fuzzer, benchmark): ' + f'({self.fuzzer}, {self.benchmark}).') def generate_coverage_summary_json(self): """Generates the coverage summary json from merged profdata file.""" diff --git a/experiment/runner.py b/experiment/runner.py index b8c95fcca..6715c51e0 100644 --- a/experiment/runner.py +++ b/experiment/runner.py @@ -180,7 +180,8 @@ def run_fuzzer(max_total_time, log_filename): target_binary = fuzzer_utils.get_fuzz_target_binary(FUZZ_TARGET_DIR, fuzz_target_name) if not target_binary: - logs.error('Fuzz target binary not found.') + logs.error(f'Fuzz target binary {fuzz_target_name} not found under ' + f'{FUZZ_TARGET_DIR}') return if max_total_time is None: From c82ace471e67f71a21e495603f7ed6e32394e24a Mon Sep 17 00:00:00 2001 From: Dongge Liu Date: Sun, 25 Aug 2024 09:34:23 +1000 Subject: [PATCH 14/15] More logs for cloud build --- experiment/build/gcb_build.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/experiment/build/gcb_build.py b/experiment/build/gcb_build.py index f553848c7..8030f23dd 100644 --- a/experiment/build/gcb_build.py +++ b/experiment/build/gcb_build.py @@ -105,7 +105,8 @@ def _build( # TODO(metzman): Refactor code so that local_build stores logs as well. build_utils.store_build_logs(config_name, result) if result.retcode != 0: - logs.error('%s failed.', command) + logs.error('%s failed. Return code: %d. Output: %s. Timedout: %s', + command, result.retcode, result.output, result.timed_out) raise subprocess.CalledProcessError(result.retcode, command) return result From 6d972e877a65455190e941537ea6030ad5ef61e0 Mon Sep 17 00:00:00 2001 From: Dongge Liu Date: Sun, 25 Aug 2024 09:47:21 +1000 Subject: [PATCH 15/15] More logs to help debug no binary --- common/fuzzer_utils.py | 6 ++++++ experiment/runner.py | 2 ++ 2 files changed, 8 insertions(+) diff --git a/common/fuzzer_utils.py b/common/fuzzer_utils.py index a28acf5db..b47e2860f 100644 --- a/common/fuzzer_utils.py +++ b/common/fuzzer_utils.py @@ -72,6 +72,12 @@ def dockerfiles(self): def get_fuzz_target_binary(search_directory: str, fuzz_target_name: str) -> Optional[str]: """Return target binary path.""" + logs.info(f'Searching for fuzz target binary named {fuzz_target_name} under' + f' directory {search_directory}') + logs.info(f'Search diretory {os.path.abspath(search_directory)} exists: ' + f'{os.path.exists(os.path.abspath(search_directory))}') + logs.info(f'list Search diretory {search_directory}: ' + f'{os.listdir(search_directory)}') if fuzz_target_name: fuzz_target_binary = os.path.join(search_directory, fuzz_target_name) if os.path.exists(fuzz_target_binary): diff --git a/experiment/runner.py b/experiment/runner.py index 6715c51e0..776a4656e 100644 --- a/experiment/runner.py +++ b/experiment/runner.py @@ -177,6 +177,8 @@ def run_fuzzer(max_total_time, log_filename): input_corpus = environment.get('SEED_CORPUS_DIR') output_corpus = os.environ['OUTPUT_CORPUS_DIR'] fuzz_target_name = environment.get('FUZZ_TARGET') + logs.info('all ENV VAR ' + f'{[f"{key}: {value}" for key, value in os.environ.items()]}') target_binary = fuzzer_utils.get_fuzz_target_binary(FUZZ_TARGET_DIR, fuzz_target_name) if not target_binary: