diff --git a/.github/workflows/presidecms.yml b/.github/workflows/presidecms.yml index f234c54..bd37bb6 100644 --- a/.github/workflows/presidecms.yml +++ b/.github/workflows/presidecms.yml @@ -53,6 +53,7 @@ jobs: needs: [ setup-presidecms ] strategy: fail-fast: false + max-parallel: 1 matrix: luceeVersion: [ 5.4/snapshot/jar, 6.0/snapshot/jar, 6.1/snapshot/jar, 6.1.0/snapshot/jar, 6.2/snapshot/jar ] javaVersion: [ 11, 21 ] @@ -109,3 +110,51 @@ jobs: PRESIDETEST_DB_PASSWORD: preside PRESIDETEST_DB_NAME: preside_test LUCEE_BASE_CONFIG: ${{ github.workspace }}/tests/.cfconfig.json + - uses: actions/upload-artifact@v4 + if: always() + with: + name: results-${{ strategy.job-index }} + path: ${{ github.workspace }}/tests/artifacts + + report: + runs-on: ubuntu-latest + if: always() + needs: [ run-preside-tests ] + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: 11 + distribution: "temurin" + - name: Cache Maven packages + if: always() + uses: actions/cache@v4 + with: + path: ~/.m2 + key: maven-cache + - name: Cache Lucee files + uses: actions/cache@v4 + if: always() + with: + path: /home/runner/work/_actions/lucee/script-runner/main/lucee-download-cache + key: lucee-downloads + - uses: actions/download-artifact@v4 + with: + path: ${{ github.workspace }}/custom/spreadsheet-cfml/test/artifacts + merge-multiple: true + pattern: results-* + - name: Display structure of downloaded files + run: ls -R ${{ github.workspace }}/custom/spreadsheet-cfml/test/artifacts + - name: Generate Report + uses: lucee/script-runner@main + with: + webroot: ${{ github.workspace }}/custom/spreadsheet-cfml/test + execute: /report.cfm + luceeVersionQuery: 6/stable/light + - uses: actions/upload-artifact@v4 + if: always() + with: + name: all-results + path: ${{ github.workspace }}/custom/spreadsheet-cfml/test/artifacts \ No newline at end of file diff --git a/.github/workflows/spreadsheet-cfml.yml b/.github/workflows/spreadsheet-cfml.yml index bcc483b..b8aafdb 100644 --- a/.github/workflows/spreadsheet-cfml.yml +++ b/.github/workflows/spreadsheet-cfml.yml @@ -89,3 +89,52 @@ jobs: execute: /test/index.cfm luceeVersionQuery: ${{ matrix.luceeVersion }} compile: ${{ env.compile }} + - uses: actions/upload-artifact@v4 + if: always() + with: + name: results-${{ strategy.job-index }} + path: ${{ github.workspace }}/test/artifacts + + report: + runs-on: ubuntu-latest + if: always() + needs: [ tests ] + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: 11 + distribution: "temurin" + - name: Cache Maven packages + if: always() + uses: actions/cache@v4 + with: + path: ~/.m2 + key: maven-cache + - name: Cache Lucee files + uses: actions/cache@v4 + if: always() + with: + path: /home/runner/work/_actions/lucee/script-runner/main/lucee-download-cache + key: lucee-downloads + - uses: actions/download-artifact@v4 + with: + path: ${{ github.workspace }}/custom/spreadsheet-cfml/test/artifacts + merge-multiple: true + pattern: results-* + - name: Display structure of downloaded files + run: ls -R ${{ github.workspace }}/custom/spreadsheet-cfml/test/artifacts + - name: Generate Report + uses: lucee/script-runner@main + with: + webroot: ${{ github.workspace }}/custom/spreadsheet-cfml/test + execute: /report.cfm + luceeVersionQuery: 6/stable/light + - uses: actions/upload-artifact@v4 + if: always() + with: + name: all-results + path: ${{ github.workspace }}/custom/spreadsheet-cfml/test/artifacts + diff --git a/.gitignore b/.gitignore index cc7222d..5c90e7d 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ /custom/benchmark/artifacts +/custom/spreadsheet-cfml/test/artifacts +/custom/presidecms/tests/artifacts diff --git a/custom/presidecms/tests/runtests.cfm b/custom/presidecms/tests/runtests.cfm index 8561a91..8065cd2 100644 --- a/custom/presidecms/tests/runtests.cfm +++ b/custom/presidecms/tests/runtests.cfm @@ -320,6 +320,21 @@ return ArrayToList( dbDesc, ", " ); } + testRunner = New testbox.system.TestBox(); + dir = getDirectoryFromPath( getCurrentTemplatePath() ) & "artifacts/"; + if (!directoryExists( dir )) + directoryCreate( dir ); + reporter = testRunner.buildReporter( "json" ); + reportFile = dir & server.lucee.version & "-" & server.java.version & "-results.json"; + systemOutput( "Writing testbox stats to #reportFile#", true ); + + report = reporter.runReport( results=result, testbox=testRunner, justReturn=true ); + report = deserializeJSON(report); + report["javaVersion"] = server.java.version; + + fileWrite( reportFile, serializeJson(report) ); + + results = []; results_md = ["## Preside CMS, Lucee #server.lucee.version# / Java #server.java.version#", ""]; diff --git a/custom/spreadsheet-cfml/test/index.cfm b/custom/spreadsheet-cfml/test/index.cfm index c0e5fa9..75ee918 100644 --- a/custom/spreadsheet-cfml/test/index.cfm +++ b/custom/spreadsheet-cfml/test/index.cfm @@ -21,6 +21,19 @@ // #(failure?':x:':':heavy_check_mark:')# systemOutput( report, true ); + dir = getDirectoryFromPath( getCurrentTemplatePath() ) & "artifacts/"; + if (!directoryExists( dir )) + directoryCreate( dir ); + reporter = testRunner.buildReporter( "json" ); + reportFile = dir & server.lucee.version & "-" & server.java.version & "-results.json"; + systemOutput( "Writing testbox stats to #reportFile#", true ); + + report = reporter.runReport( results=result, testbox=testRunner, justReturn=true ); + report = deserializeJSON(report); + report["javaVersion"] = server.java.version; + + fileWrite( reportFile, serializeJson(report) ); + exeTime = "Test Execution time: #DecimalFormat( result.getTotalDuration() /1000 )# s"; if ( structKeyExists( server.system.environment, "GITHUB_STEP_SUMMARY" ) ){ fileAppend( server.system.environment.GITHUB_STEP_SUMMARY, diff --git a/custom/spreadsheet-cfml/test/report.cfm b/custom/spreadsheet-cfml/test/report.cfm new file mode 100644 index 0000000..1a31e06 --- /dev/null +++ b/custom/spreadsheet-cfml/test/report.cfm @@ -0,0 +1,166 @@ + + dir = getDirectoryFromPath( getCurrentTemplatePath() ) & "artifacts"; + systemOutput ( dir, true ); + files = directoryList( path=dir, filter="*-results.json" ); + + runs = []; + + for ( f in files ){ + systemOutput ( f, true ); + json = deserializeJson( fileRead( f ) ); + q = queryNew( "time,suite,spec,suiteSpec" ); + + for ( i=1; i <= len(json.bundleStats); i++ ){ + bundle = json.bundleStats[i]; + for (j = 1; j <= len(bundle.suiteStats); j++){ + s = bundle.suiteStats[ j ]; + for ( p in s.specStats ){ + row = queryAddRow( q ); + // querySetCell(q, "java", json.javaVersion, row); + // querySetCell(q, "version", json.CFMLEngineVersion, row); + querySetCell(q, "spec", p.name, row); + querySetCell(q, "suite", s.name, row); + querySetCell(q, "suiteSpec", s.name & ", " & p.name, row); + querySetCell(q, "time", p.totalDuration, row); + } + } + } + + arrayAppend( runs, { + "java": json.javaVersion, + "version": json.CFMLEngineVersion, + "totalDuration": json.totalDuration, + "stats": queryToStruct(q, "suiteSpec") + }); + }; + + if ( IsEmpty( runs ) ) throw "No json report files found?"; + + _logger( "## Summary Report" ); + + reportRuns( runs ); + + reportTests( runs ); + + function _logger( string message="", boolean throw=false ){ + if ( !structKeyExists( server.system.environment, "GITHUB_STEP_SUMMARY" ) ){ + systemOutput( arguments.message, true ); + return; + } + + if ( !FileExists( server.system.environment.GITHUB_STEP_SUMMARY ) ){ + fileWrite( server.system.environment.GITHUB_STEP_SUMMARY, "#### #server.lucee.version# "); + //fileAppend( server.system.environment.GITHUB_STEP_SUMMARY, server.system.environment.toJson()); + } + + if ( arguments.throw ) { + fileAppend( server.system.environment.GITHUB_STEP_SUMMARY, "> [!WARNING]" & chr(10) ); + fileAppend( server.system.environment.GITHUB_STEP_SUMMARY, "> #arguments.message##chr(10)#"); + throw arguments.message; + } else { + fileAppend( server.system.environment.GITHUB_STEP_SUMMARY, "#arguments.message##chr(10)#"); + } + + } + + function reportRuns( srcRuns ) localmode=true { + + var runs = duplicate( srcRuns ); + arraySort( + runs, + function (e1, e2){ + if (e1.totalDuration lt e2.totalDuration) return -1; + else if (e1.totalDuration gt e2.totalDuration) return 1; + return 0; + } + ); // fastest to slowest + + var hdr = [ "Version", "Java", "Time" ]; + var div = [ "---", "---", "---:" ]; + _logger( "" ); + _logger( "|" & arrayToList( hdr, "|" ) & "|" ); + _logger( "|" & arrayToList( div, "|" ) & "|" ); + + var row = []; + loop array=runs item="local.run" { + ArrayAppend( row, run.version ); + ArrayAppend( row, run.java ); + arrayAppend( row, numberFormat( run.totalDuration ) ); + _logger( "|" & arrayToList( row, "|" ) & "|" ); + row = []; + } + + _logger( "" ); + } + + + function reportTests( runs ) localmode=true { + + var sortedRuns = duplicate(runs); + + arraySort( + sortedRuns, + function (e1, e2){ + return compare(e1.version & e1.java, e2.version & e2.java); + } + ); // sort runs by oldest version to newest version + + + var hdr = [ "Suite / Spec" ]; + var div = [ "---" ]; + loop array=sortedRuns item="local.run" { + arrayAppend( hdr, run.version & " " & listFirst(run.java,".") ); + arrayAppend( div, "---:" ); // right align as they are all numeric + } + + // diff column, first run vs last run + arrayAppend( hdr, "Difference (oldest vs newest)" ); + arrayAppend( div, "---:" ); // right align as they are all numeric + + _logger( "" ); + _logger( "|" & arrayToList( hdr, "|" ) & "|" ); + _logger( "|" & arrayToList( div, "|" ) & "|" ); + + // now sort the tests by the difference in time between the first run and last run + + var suiteSpecs = []; + var suiteSpecsDiff = {}; + + loop collection=runs[1].stats key="title" value="test" { + // difference between the test time for the newest version minus oldest version + var diff = sortedRuns[arrayLen(runs)].stats[test.suiteSpec].time - sortedRuns[1].stats[test.suiteSpec].time; + ArrayAppend( suiteSpecs, { + suiteSpec: test.suiteSpec, + diff: diff + }); + suiteSpecsDiff[test.suiteSpec] = diff; + } + + arraySort( + suiteSpecs, + function (e1, e2){ + if (e1.diff gt e2.diff) return -1; + else if (e1.diff lt e2.diff) return 1; + return 0; + } + ); // sort by performance regression + + var row = []; + loop array=suiteSpecs item="test" { + // force long names to wrap without breaking markdown + ArrayAppend( row, REReplace( wrap(test.suiteSpec, 70), "\n", " ", "ALL") ); + loop array=sortedRuns item="local.run" { + if ( structKeyExists( run.stats, test.suiteSpec ) ) + arrayAppend( row, numberFormat( run.stats[test.suiteSpec].time ) ); + else + arrayAppend( row, ""); + } + arrayAppend( row, numberFormat( test.diff ) ); + _logger( "|" & arrayToList( row, "|" ) & "|" ); + row = []; + } + + _logger( "" ); + } + + \ No newline at end of file