diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..830ce5d2 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,31 @@ +--- +name: Report an issue +about: Create a bug report to fix an existing issue. +title: '' +labels: 'bug' +assignees: '' + +--- + +### Description + +> Provide a description of the issue + +### Expected Behavior + +### Actual Behavior + +### Reproduction + +> Detail the steps taken to reproduce the issue +> +> Where applicable, please include (exclude sensitive information): +> +> - Code of Files to reproduce the issue +> - Log files +> - Application settings +> - Screenshots + +### Environment Details + +> Provide any information relating to the environment the issue was identified in - include applicable version and additional runtime information (include OS or other underlying infrastructure) diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000..99f9dee4 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest new functionality for this project. +title: '' +labels: 'enhancement' +assignees: '' + +--- + +### Describe the problem + +> A clear description of what the problem is. + +### Proposed solution + +> A clear description of what you want to happen. + +### Additional details + +> Add any other details / contexts / screenshots about the feature request. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..14f3411d --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,21 @@ +By submitting a PR to this repository, you agree to the terms within the [Checkmarx Code of Conduct](https://github.com/checkmarx-ltd/open-source-template/blob/master/CODE-OF-CONDUCT.md). Please see the [contributing guidelines](https://github.com/checkmarx-ltd/open-source-template/blob/master/CONTRIBUTING.md) for how to create and submit a high-quality PR for this repo. + +### Description + +> Describe the purpose of this PR along with any background information and the impacts of the proposed change. + +### References + +> Include supporting link to GitHub Issue/PR number + +### Testing + +> Describe how this change was tested. Be specific about anything not tested and reasons why. If this solution has unit and/or integration testing, tests should be added for new functionality and existing tests should complete without errors. +> +> Please include any manual steps for testing end-to-end or functionality not covered by unit/integration tests. + +### Checklist + +- [ ] I have added documentation for new/changed functionality in this PR (if applicable). *If documentaiton is a Wiki Update, please indicate desired changes within PR MD Comment* +- [ ] All active GitHub checks for tests, formatting, and security are passing +- [ ] The correct base branch is being used diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml new file mode 100644 index 00000000..acf275b6 --- /dev/null +++ b/.github/release-drafter.yml @@ -0,0 +1,32 @@ +name-template: 'v$RESOLVED_VERSION 🌈' +tag-template: 'v$RESOLVED_VERSION' +categories: + - title: '🚀 Features' + labels: + - 'feature' + - 'enhancement' + - title: '🐛 Bug Fixes' + labels: + - 'fix' + - 'bugfix' + - 'bug' + - title: '🧰 Maintenance' + label: 'chore' +exclude-labels: + - 'skip-release-notes' +change-template: '- $TITLE @$AUTHOR (#$NUMBER)' +version-resolver: + major: + labels: + - 'major' + minor: + labels: + - 'minor' + patch: + labels: + - 'patch' + default: patch +template: | + ## Changes + + $CHANGES diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml new file mode 100644 index 00000000..8ea20312 --- /dev/null +++ b/.github/workflows/release-drafter.yml @@ -0,0 +1,19 @@ +name: Release Drafter + +on: + push: + # branches to consider in the event; optional, defaults to all + branches: + - master + +jobs: + update_release_draft: + runs-on: ubuntu-latest + steps: + # Drafts your next Release notes as Pull Requests are merged into "master" + - uses: release-drafter/release-drafter@v5 + with: + # (Optional) specify config name to use, relative to .github/. Default: release-drafter.yml + config-name: release-drafter.yml + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 6a755137..a9e58c3f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ work/ out/ +*-secrets.properties #Gradle .gradle/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..780d9703 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,77 @@ +# Contributing to Checkmarx projects + +Welcome and thank you for considering contributing to a Checkmarx open source project. + +Reading and following these guidelines will help us make the contribution process easy and effective for everyone involved. It also communicates that you agree to respect the time of the developers managing and developing these open source projects. In return, we will reciprocate that respect by addressing your issue, assessing changes, and helping you finalize your pull requests. + + +## Quicklinks + +- [Contributing to Checkmarx projects](#contributing-to-checkmarx-projects) + - [Quicklinks](#quicklinks) + - [Code of Conduct](#code-of-conduct) + - [Getting Started](#getting-started) + - [Issues](#issues) + - [Pull Requests](#pull-requests) + - [Resources](#resources) + +## Code of Conduct + +By participating and contributing to any Checkmarx projects, you agree to uphold our [Code of Conduct](https://github.com/checkmarx-ltd/open-source-template/blob/master/CODE-OF-CONDUCT.md). + +## Getting Started + +If you have suggestions for how this project could be improved, or want to report a bug, open an issue. We appreciate all contributions. If you have questions, we'd love to hear them. + +We also appreciate PRs. If you're thinking of submitting any PR, pleae open an issue first to spark a discussion around it. + +Contributions are made to this repo via Issues and Pull Requests (PRs). A few general guidelines that cover both: + +- Search for existing Issues and PRs before creating your own to avoid duplicates. +- PRs will only be accepted if associated with an issue (enhancement or bug) that has been submitted and reviewed/labeld as *accepted* by a Checkmarx team member. +- We will work hard to makes sure issues that are raised are handled in a timely manner. + +## Issues + +Issues should be used to report problems with the solution / source code, request a new feature, or to discuss potential changes before a PR is created. When you create a new Issue, a template will be loaded that will guide you through collecting and providing the information we need to investigate. + +If you find an Issue that addresses the problem you're having, please add your own reproduction information to the existing issue rather than creating a new one. Adding a [reaction](https://github.blog/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/) can also help by indicating to our maintainers that a particular problem is affecting more than just the reporter. + +### Templates + +The following templates will be used within Checkmarx github repositories +- [Enhancement/Feature Request Template](.github/ISSUE_TEMPLATE/feature_request.md) +- [Bug Report Template](.github/ISSUE_TEMPLATE/bug_report.md) + +## Pull Requests + +PRs to our source are always welcome and can be a quick way to get your fix or improvement slated for the next release. In general, PRs should: + +- Only fix/add the functionality in question **or** address code style issues, not both. +- Ensure all necessary details are provided and adhered to +- Add unit or integration tests for fixed or changed functionality (if a test suite already exists) or specify steps taken to ensure changes were tested and functionality works as expected. +- Address a single concern in the least number of changed lines as possible. +- Include documentation in the repo or Provide additional comments in Markdown comments that should be pulled/reflected in GitHub Wiki for the given project. +- Be accompanied by a complete Pull Request template (loaded automatically when a PR is created). + +For changes that address core functionality or would require breaking changes (e.g. a major release), please open an Issue to discuss your proposal first. + +In general, we follow the _fork-and-pull_ Git workflow + +1. Fork the repository to your own Github account +2. Clone the project to your machine +3. Create a branch locally with a succinct but descriptive name (prefix with feature/-descriptive-name> or hotfix/-descriptive-name) +4. Commit changes to the branch +5. Push changes to your fork +6. Open a PR in our repository and follow the PR template so that we can efficiently review and asses the changes. *Ensure an associated Issue has been accepted by the Checkmarx team.* + +### Templates +The following template will be used within Checkmarx github repositories + +[Pull Request Template](.github/PULL_REQUEST_TEMPLATE.md) + +## Resources + +- [How to Contribute to Open Source](https://opensource.guide/how-to-contribute/) +- [Using Pull Requests](https://help.github.com/articles/about-pull-requests/) +- [GitHub Help](https://help.github.com) diff --git a/Jenkinsfile-CI-CD b/Jenkinsfile-CI-CD new file mode 100644 index 00000000..a8aebb9d --- /dev/null +++ b/Jenkinsfile-CI-CD @@ -0,0 +1,548 @@ +#!/groovy +@Library('cx-jenkins-pipeline-kit') _ +import java.time.* + +def workspace +def vmName = "${BUILD_TAG}-CxSAST" +def vmTemplate89 = "CxSDLC-Template-CxSAST-8-9" +def ipAddress89 +def vmTemplate90 = "CxSDLC-Template-CxSAST-9-0" +def ipAddress90 +def vmTemplate92 = "CxSDLC-Template-CxSAST-9-2" +def ipAddress92 +def vmTemplate93 = "CxSDLC-Template-CxSAST-9-3" +def ipAddress93 +def ram = "12000" +def cpu = "8" +def provider = "VMWARE" +def decommissionPeriod = "3 hour" +def vmwareNetwork = "Lab" +def automationBranch = "9.0.0" + +pipeline { + parameters { + string(name: "vmTemplate89",defaultValue: "${vmTemplate89}", description: "Template for 8.9 VM") + string(name: "vmTemplate90",defaultValue: "${vmTemplate90}", description: "Template for 9.0 VM") + string(name: "vmTemplate92",defaultValue: "${vmTemplate92}", description: "Template for 9.2 VM") + string(name: "vmTemplate93",defaultValue: "${vmTemplate93}", description: "Template for 9.3 VM") + string(name: "ram",defaultValue: "${ram}", description: "Server memory") + string(name: "cpu",defaultValue: "${cpu}", description: "") + string(name: "provider",defaultValue: "${provider}", description: "IAAS platform to be used") + string(name: "decommissionPeriod",defaultValue: "${decommissionPeriod}", description: "Decommission period") + string(name: "vmwareNetwork",defaultValue: "${vmwareNetwork}", description: "vmware network for new VMs") + string(name: "automationBranch", defaultValue: "${automationBranch}", description: "automation branch") + } + agent { node { label 'CxSDLC-Slave' } } + options { + timestamps() + timeout(time: 2, unit: 'HOURS') + //skipDefaultCheckout() + } + stages { + + stage('Pipeline Info') { + steps { + script { + env.PIPELINE_STATUS = "Success" + env.STAGE_NAME_FAILED = "None" + if (BRANCH_NAME == 'master') { + Calendar cal = Calendar.getInstance(Locale.US) + int quarter = (cal.get(Calendar.MONTH) / 3) + 1 + int year = cal.get(Calendar.YEAR) + env.cxCommonVersion = "${year}.${quarter}.${BUILD_NUMBER}" + sh "sed -e 's/\${cxcommon.version}/${env.cxCommonVersion}/g' -i ./pom.xml" + // get version from POM + /*sh "mvn resources:resources" + def commonPropertiesContent = readFile "./target/classes/common.properties" + env.cxCommonVersion = commonPropertiesContent.substring(10)*/ + } else { + env.cxCommonVersion = "${BUILD_TAG}" + } + workspace = pwd() + if (env.automationBranch == null) { + env.automationBranch = automationBranch + } + sh 'printenv' + } + } + } + + stage('Build CxCommon') { + steps { + script { + sh "docker run --rm --name build-${BUILD_TAG} -v /root/.m2:/root/.m2 -v ${workspace}:/usr/src/cx-common -w /usr/src/cx-common maven:3.6.1-jdk-8-alpine mvn clean install -DskipTests -Dcxcommon.version=${env.cxCommonVersion}" + } + } + post { + failure { + script { + env.PIPELINE_STATUS = "Failure" + env.STAGE_NAME_FAILED="${STAGE_NAME}" + } + } + } + } + + stage('UT-IT & Sonar') { + parallel { + + stage('Run Unit & Integration Tests') { + steps { + script { + sh "docker run --rm --name test-${BUILD_TAG} -v /root/.m2:/root/.m2 -v ${workspace}:/usr/src/cx-common -w /usr/src/cx-common maven:3.6.1-jdk-8-alpine mvn test -Dcxcommon.version=${env.cxCommonVersion}" + } + } + post { + failure { + script { + env.PIPELINE_STATUS = "Failure" + env.STAGE_NAME_FAILED="${STAGE_NAME}" + } + } + } + } + + stage('Run Code Quality') { + environment { + SONAR_CLOUD_TOKEN = credentials('sonarcloud') + } + steps { + script { + sh "docker run --rm --name sonar-${BUILD_TAG} -v /root/.m2:/root/.m2 -v ${workspace}:/usr/src/cx-common -w /usr/src/cx-common maven:3.6.3-jdk-11-slim mvn sonar:sonar -Dcxcommon.version=${env.cxCommonVersion} -Dsonar.login=${SONAR_CLOUD_TOKEN}" + def sonarTaskUrl = sh (returnStdout: true, script: "awk '/./{line=\$0} END{print line}' ./target/sonar/report-task.txt | cut -d '=' -f '2,3'").trim() + def sonarTaskStatus = sh (returnStdout: true, script: "curl -s -u ${SONAR_CLOUD_TOKEN}: ${sonarTaskUrl} | jq -r '.task.status'").trim() + while (sonarTaskStatus == 'PENDING' || sonarTaskStatus == 'IN_PROGRESS') { + echo "Sonar task status is: ${sonarTaskStatus}. Waiting for 5 seconds..." + sleep(5) + sonarTaskStatus = sh (returnStdout: true, script: "curl -s -u ${SONAR_CLOUD_TOKEN}: ${sonarTaskUrl} | jq -r '.task.status'").trim() + } + if (sonarTaskStatus == "FAILED" || sonarTaskStatus == "CANCELED") { + Error_Msg("Sonar scan ${sonarTaskStatus}. You can find more details at https://sonarcloud.io/dashboard?id=checkmarx-ltd_Cx-Client-Common") + error "Sonar scan ${sonarTaskStatus}. You can find more details at https://sonarcloud.io/dashboard?id=checkmarx-ltd_Cx-Client-Common" + } + def sonarAnalysisId = sh (returnStdout: true, script: "curl -s -u ${SONAR_CLOUD_TOKEN}: ${sonarTaskUrl} | jq -r '.task.analysisId'").trim() + def sonarResultsStatus = sh (returnStdout: true, script: "curl -s -u ${SONAR_CLOUD_TOKEN}: https://sonarcloud.io/api/qualitygates/project_status?analysisId=${sonarAnalysisId} | jq -r '.projectStatus.status'").trim() + if (sonarResultsStatus == "ERROR") { + kit.Error_Msg("Sonar scan FAILED. You can find more details at https://sonarcloud.io/dashboard?id=checkmarx-ltd_Cx-Client-Common") + error "Sonar scan FAILED. You can find more details at https://sonarcloud.io/dashboard?id=checkmarx-ltd_Cx-Client-Common" + } + } + } + post { + failure { + script { + env.PIPELINE_STATUS = "Failure" + env.STAGE_NAME_FAILED="${STAGE_NAME}" + } + } + } + } + } + } + + stage('Install CxCommon in Local Repository') { + when { + expression { + BRANCH_NAME == 'master' || BRANCH_NAME.startsWith("PR-") && CHANGE_TARGET == 'master' + } + } + steps { + sh "docker run --rm --memory 2gb --name install-cx-common-${BUILD_TAG} -v maven-repo:/root/.m2 -v ${workspace}/target/cx-client-common-${env.cxCommonVersion}.jar:/usr/src/artifact.jar maven:3.6.1-jdk-8-alpine \ + mvn -q install:install-file -Dfile=/usr/src/artifact.jar -DgroupId=com.checkmarx -DartifactId=cx-client-common -Dversion=${env.cxCommonVersion} -Dpackaging=jar" + sh "docker run --rm --memory 2gb --name install-cx-common-${BUILD_TAG} -v maven-repo:/root/.m2 -v ${workspace}/pom.xml:/usr/src/pom.xml maven:3.6.1-jdk-8-alpine \ + mvn -q install:install-file -Dfile=/usr/src/pom.xml -DgroupId=com.checkmarx -DartifactId=cx-client-common -Dversion=${env.cxCommonVersion} -Dpackaging=pom" + } + } + + stage('System Tests') { + when { + expression { + BRANCH_NAME == 'master' || BRANCH_NAME.startsWith("PR-") && CHANGE_TARGET == 'master' + } + } + parallel { + + stage('8.9') { + when { + expression { false } + } + stages { + stage('Create VM') { + steps { + script { + kit.Create_Vm_Terraform(vmName + "-8.9", vmTemplate89, ram, cpu, provider, decommissionPeriod, "Auto", "Plugins-CI", vmwareNetwork) + ipAddress89 = kit.getIpAddress(vmName + "-8.9", provider) + node('install01') { + kit.Create_Jenkins_Slave_On_Master(vmName + "-8.9") + kit.Start_Jenkins_Slave_On_Windows_Pstools(ipAddress89, vmName + "-8.9") + } + } + } + } + stage('Pull Automation Code') { + steps { + dir("${workspace}/8.9/Checkmarx-System-Test") { + git branch: "${env.automationBranch}", credentialsId: '15f8e7b7-6ce7-44c0-b151-84f99ffa7aed', poll: false, url: 'http://tfs2013:8080/tfs/DefaultCollection/Automation/_git/Checkmarx-System-Test' + sh "cp -r ../../../env ." + sh "sed -e 's//${ipAddress89}/g' -i ./env/topology.xml" + } + } + } + stage('Plugins API Sanity Test') { + steps { + dir("${workspace}/8.9/Checkmarx-System-Test") { + sh "docker run --rm --memory 2gb --name plugins-api-sanity-test-8.9-${BUILD_TAG} -v maven-repo:/root/.m2 -v ${workspace}/8.9/Checkmarx-System-Test:/usr/src/automation -w /usr/src/automation --add-host WIN2012-ENV9-B:${ipAddress89} maven:3.6.1-jdk-8-alpine \ + mvn -q clean test -Dcxcommon.version=${env.cxCommonVersion} -Dsast.version=8.9 -Dtest=com.cx.automation.plugin.test.cxcommonclient.sanity.* -Dtopology.xml.ref=/usr/src/automation/env/topology.xml -Denv=hardening_env -DfailIfNoTests=false -Dmaven.test.failure.ignore=false -DskipTests=false" + } + } + } + stage('Plugins API Smoke Tests') { + steps { + dir("${workspace}/8.9/Checkmarx-System-Test") { + sh "docker run --rm --memory 2gb --name plugins-api-smoke-tests-8.9-${BUILD_TAG} -v maven-repo:/root/.m2 -v ${workspace}/8.9/Checkmarx-System-Test:/usr/src/automation -w /usr/src/automation --add-host WIN2012-ENV9-B:${ipAddress89} maven:3.6.1-jdk-8-alpine \ + mvn -q test -Dcxcommon.version=${env.cxCommonVersion} -Dsast.version=8.9 -Dtest=com.cx.automation.plugin.test.cxcommonclient.PluginsCxSASTSmokeTests,com.cx.automation.plugin.test.cxcommonclient.PluginsCxMandOAndOSASmokeTests \ + -Dtopology.xml.ref=/usr/src/automation/env/topology.xml -Denv=hardening_env -DfailIfNoTests=false -Dmaven.test.failure.ignore=false -DskipTests=false" + } + } + } + stage('Plugins CxCommonClient All Tests') { + steps { + dir("${workspace}/8.9/Checkmarx-System-Test") { + sh "docker run --rm --memory 2gb --name plugins-cxcommonclient-all-tests-8.9-${BUILD_TAG} -v maven-repo:/root/.m2 -v ${workspace}/8.9/Checkmarx-System-Test:/usr/src/automation -w /usr/src/automation --add-host WIN2012-ENV9-B:${ipAddress89} maven:3.6.1-jdk-8-alpine \ + mvn -q test -Dcxcommon.version=${env.cxCommonVersion} -Dsast.version=8.9 -Dtest=com.cx.automation.plugin.test.cxcommonclient.scan.*,com.cx.automation.plugin.test.cxcommonclient.osa.* \ + -Dtopology.xml.ref=/usr/src/automation/env/topology.xml -Denv=hardening_env -DfailIfNoTests=false -Dmaven.test.failure.ignore=false -DskipTests=false" + } + } + } + } + post { + always { + script { + dir("${workspace}/8.9/Checkmarx-System-Test") { + junit '**/PluginsCommonClient/target/surefire-reports/**/*.xml' + } + if (ipAddress89 != null) { + node(vmName + "-8.9") { + kit.zipStashInstallationLogs("CxSAST-8.9-Logs") + } + unstash "CxSAST-8.9-Logs" + deleteVm(provider, ipAddress89, vmName + "-8.9") + } + } + } + failure { + script { + env.PIPELINE_STATUS = "Failure" + env.STAGE_NAME_FAILED="${STAGE_NAME}" + } + } + } + } + + stage('9.0') { + stages { + stage('Create VM') { + steps { + script { + kit.Create_Vm_Terraform(vmName + "-9.0", vmTemplate90, ram, cpu, provider, decommissionPeriod, "Auto", "Plugins-CI", vmwareNetwork) + ipAddress90 = kit.getIpAddress(vmName + "-9.0", provider) + node('install01') { + kit.Create_Jenkins_Slave_On_Master(vmName + "-9.0") + kit.Start_Jenkins_Slave_On_Windows_Pstools(ipAddress90, vmName + "-9.0") + } + } + } + } + stage('Pull Automation Code') { + steps { + dir("${workspace}/9.0/Checkmarx-System-Test") { + git branch: "${env.automationBranch}", credentialsId: '15f8e7b7-6ce7-44c0-b151-84f99ffa7aed', poll: false, url: 'http://tfs2013:8080/tfs/DefaultCollection/Automation/_git/Checkmarx-System-Test' + sh "cp -r ../../../env ." + sh "sed -e 's//${ipAddress90}/g' -i ./env/topology.xml" + } + } + } + stage('Plugins API Sanity Test') { + steps { + dir("${workspace}/9.0/Checkmarx-System-Test") { + sh "docker run --rm --memory 2gb --name plugins-api-sanity-test-9.0-${BUILD_TAG} -v maven-repo:/root/.m2 -v ${workspace}/9.0/Checkmarx-System-Test:/usr/src/automation -w /usr/src/automation --add-host WIN2012-ENV9-B:${ipAddress90} maven:3.6.1-jdk-8-alpine \ + mvn -q clean test -Dcxcommon.version=${env.cxCommonVersion} -Dsast.version=9.0 -Dtest=com.cx.automation.plugin.test.cxcommonclient.sanity.* -Dtopology.xml.ref=/usr/src/automation/env/topology.xml -Denv=hardening_env -DfailIfNoTests=false -Dmaven.test.failure.ignore=false -DskipTests=false" + } + } + } + stage('Plugins API Smoke Tests') { + steps { + dir("${workspace}/9.0/Checkmarx-System-Test") { + sh "docker run --rm --memory 2gb --name plugins-api-smoke-tests-9.0-${BUILD_TAG} -v maven-repo:/root/.m2 -v ${workspace}/9.0/Checkmarx-System-Test:/usr/src/automation -w /usr/src/automation --add-host WIN2012-ENV9-B:${ipAddress90} maven:3.6.1-jdk-8-alpine \ + mvn -q test -Dcxcommon.version=${env.cxCommonVersion} -Dsast.version=9.0 -Dtest=com.cx.automation.plugin.test.cxcommonclient.PluginsCxSASTSmokeTests,com.cx.automation.plugin.test.cxcommonclient.PluginsCxMandOAndOSASmokeTests \ + -Dtopology.xml.ref=/usr/src/automation/env/topology.xml -Denv=hardening_env -DfailIfNoTests=false -Dmaven.test.failure.ignore=false -DskipTests=false" + } + } + } + stage('Plugins CxCommonClient All Tests') { + steps { + dir("${workspace}/9.0/Checkmarx-System-Test") { + sh "docker run --rm --memory 2gb --name plugins-cxcommonclient-all-tests-9.0-${BUILD_TAG} -v maven-repo:/root/.m2 -v ${workspace}/9.0/Checkmarx-System-Test:/usr/src/automation -w /usr/src/automation --add-host WIN2012-ENV9-B:${ipAddress90} maven:3.6.1-jdk-8-alpine \ + mvn -q test -Dcxcommon.version=${env.cxCommonVersion} -Dsast.version=9.0 -Dtest=com.cx.automation.plugin.test.cxcommonclient.scan.*,com.cx.automation.plugin.test.cxcommonclient.osa.* \ + -Dtopology.xml.ref=/usr/src/automation/env/topology.xml -Denv=hardening_env -DfailIfNoTests=false -Dmaven.test.failure.ignore=false -DskipTests=false" + } + } + } + } + post { + always { + script { + dir("${workspace}/9.0/Checkmarx-System-Test") { + junit '**/PluginsCommonClient/target/surefire-reports/**/*.xml' + } + if (ipAddress90 != null) { + node(vmName + "-9.0") { + kit.zipStashInstallationLogs("CxSAST-9.0-Logs") + } + unstash "CxSAST-9.0-Logs" + deleteVm(provider, ipAddress90, vmName + "-9.0") + } + } + } + failure { + script { + env.PIPELINE_STATUS = "Failure" + env.STAGE_NAME_FAILED="${STAGE_NAME}" + } + } + } + } + + stage('9.2') { + stages { + stage('Create VM') { + steps { + script { + kit.Create_Vm_Terraform(vmName + "-9.2", vmTemplate92, ram, cpu, provider, decommissionPeriod, "Auto", "Plugins-CI", vmwareNetwork) + ipAddress92 = kit.getIpAddress(vmName + "-9.2", provider) + node('install01') { + kit.Create_Jenkins_Slave_On_Master(vmName + "-9.2") + kit.Start_Jenkins_Slave_On_Windows_Pstools(ipAddress92, vmName + "-9.2") + } + } + } + } + stage('Pull Automation Code') { + steps { + dir("${workspace}/9.2/Checkmarx-System-Test") { + git branch: "${env.automationBranch}", credentialsId: '15f8e7b7-6ce7-44c0-b151-84f99ffa7aed', poll: false, url: 'http://tfs2013:8080/tfs/DefaultCollection/Automation/_git/Checkmarx-System-Test' + sh "cp -r ../../../env ." + sh "sed -e 's//${ipAddress92}/g' -i ./env/topology.xml" + } + } + } + stage('Plugins API Sanity Test') { + steps { + dir("${workspace}/9.2/Checkmarx-System-Test") { + sh "docker run --rm --memory 2gb --name plugins-api-sanity-test-9.2-${BUILD_TAG} -v maven-repo:/root/.m2 -v ${workspace}/9.2/Checkmarx-System-Test:/usr/src/automation -w /usr/src/automation --add-host WIN2012-ENV9-B:${ipAddress92} maven:3.6.1-jdk-8-alpine \ + mvn -q clean test -Dcxcommon.version=${env.cxCommonVersion} -Dsast.version=9.2 -Dtest=com.cx.automation.plugin.test.cxcommonclient.sanity.* -Dtopology.xml.ref=/usr/src/automation/env/topology.xml -Denv=hardening_env -DfailIfNoTests=false -Dmaven.test.failure.ignore=false -DskipTests=false" + } + } + } + stage('Plugins API Smoke Tests') { + steps { + dir("${workspace}/9.2/Checkmarx-System-Test") { + sh "docker run --rm --memory 2gb --name plugins-api-smoke-tests-9.2-${BUILD_TAG} -v maven-repo:/root/.m2 -v ${workspace}/9.2/Checkmarx-System-Test:/usr/src/automation -w /usr/src/automation --add-host WIN2012-ENV9-B:${ipAddress92} maven:3.6.1-jdk-8-alpine \ + mvn -q test -Dcxcommon.version=${env.cxCommonVersion} -Dsast.version=9.2 -Dtest=com.cx.automation.plugin.test.cxcommonclient.PluginsCxSASTSmokeTests,com.cx.automation.plugin.test.cxcommonclient.PluginsCxMandOAndOSASmokeTests \ + -Dtopology.xml.ref=/usr/src/automation/env/topology.xml -Denv=hardening_env -DfailIfNoTests=false -Dmaven.test.failure.ignore=false -DskipTests=false" + } + } + } + stage('Plugins CxCommonClient All Tests') { + steps { + dir("${workspace}/9.2/Checkmarx-System-Test") { + sh "docker run --rm --memory 2gb --name plugins-cxcommonclient-all-tests-9.2-${BUILD_TAG} -v maven-repo:/root/.m2 -v ${workspace}/9.2/Checkmarx-System-Test:/usr/src/automation -w /usr/src/automation --add-host WIN2012-ENV9-B:${ipAddress92} maven:3.6.1-jdk-8-alpine \ + mvn -q test -Dcxcommon.version=${env.cxCommonVersion} -Dsast.version=9.2 -Dtest=com.cx.automation.plugin.test.cxcommonclient.scan.*,com.cx.automation.plugin.test.cxcommonclient.osa.* \ + -Dtopology.xml.ref=/usr/src/automation/env/topology.xml -Denv=hardening_env -DfailIfNoTests=false -Dmaven.test.failure.ignore=false -DskipTests=false" + } + } + } + } + post { + always { + script { + dir("${workspace}/9.2/Checkmarx-System-Test") { + junit '**/PluginsCommonClient/target/surefire-reports/**/*.xml' + } + if (ipAddress92 != null) { + node(vmName + "-9.2") { + kit.zipStashInstallationLogs("CxSAST-9.2-Logs") + } + unstash "CxSAST-9.2-Logs" + deleteVm(provider, ipAddress92, vmName + "-9.2") + } + } + } + failure { + script { + env.PIPELINE_STATUS = "Failure" + env.STAGE_NAME_FAILED="${STAGE_NAME}" + } + } + } + } + + stage('9.3') { + stages { + stage('Create VM') { + steps { + script { + kit.Create_Vm_Terraform(vmName + "-9.3", vmTemplate93, ram, cpu, provider, decommissionPeriod, "Auto", "Plugins-CI", vmwareNetwork) + ipAddress93 = kit.getIpAddress(vmName + "-9.3", provider) + node('install01') { + kit.Create_Jenkins_Slave_On_Master(vmName + "-9.3") + kit.Start_Jenkins_Slave_On_Windows_Pstools(ipAddress93, vmName + "-9.3") + } + } + } + } + stage('Pull Automation Code') { + steps { + dir("${workspace}/9.3/Checkmarx-System-Test") { + git branch: "${env.automationBranch}", credentialsId: '15f8e7b7-6ce7-44c0-b151-84f99ffa7aed', poll: false, url: 'http://tfs2013:8080/tfs/DefaultCollection/Automation/_git/Checkmarx-System-Test' + sh "cp -r ../../../env ." + sh "sed -e 's//${ipAddress93}/g' -i ./env/topology.xml" + } + } + } + stage('Plugins API Sanity Test') { + steps { + dir("${workspace}/9.3/Checkmarx-System-Test") { + sh "docker run --rm --memory 2gb --name plugins-api-sanity-test-9.3-${BUILD_TAG} -v maven-repo:/root/.m2 -v ${workspace}/9.3/Checkmarx-System-Test:/usr/src/automation -w /usr/src/automation --add-host WIN2012-ENV9-B:${ipAddress93} maven:3.6.1-jdk-8-alpine \ + mvn -q clean test -Dcxcommon.version=${env.cxCommonVersion} -Dsast.version=9.3 -Dtest=com.cx.automation.plugin.test.cxcommonclient.sanity.* -Dtopology.xml.ref=/usr/src/automation/env/topology.xml -Denv=hardening_env -DfailIfNoTests=false -Dmaven.test.failure.ignore=false -DskipTests=false" + } + } + } + stage('Plugins API Smoke Tests') { + steps { + dir("${workspace}/9.3/Checkmarx-System-Test") { + sh "docker run --rm --memory 2gb --name plugins-api-smoke-tests-9.3-${BUILD_TAG} -v maven-repo:/root/.m2 -v ${workspace}/9.3/Checkmarx-System-Test:/usr/src/automation -w /usr/src/automation --add-host WIN2012-ENV9-B:${ipAddress93} maven:3.6.1-jdk-8-alpine \ + mvn -q test -Dcxcommon.version=${env.cxCommonVersion} -Dsast.version=9.3 -Dtest=com.cx.automation.plugin.test.cxcommonclient.PluginsCxSASTSmokeTests,com.cx.automation.plugin.test.cxcommonclient.PluginsCxMandOAndOSASmokeTests \ + -Dtopology.xml.ref=/usr/src/automation/env/topology.xml -Denv=hardening_env -DfailIfNoTests=false -Dmaven.test.failure.ignore=false -DskipTests=false" + } + } + } + stage('Plugins CxCommonClient All Tests') { + steps { + dir("${workspace}/9.3/Checkmarx-System-Test") { + sh "docker run --rm --memory 2gb --name plugins-cxcommonclient-all-tests-9.3-${BUILD_TAG} -v maven-repo:/root/.m2 -v ${workspace}/9.3/Checkmarx-System-Test:/usr/src/automation -w /usr/src/automation --add-host WIN2012-ENV9-B:${ipAddress93} maven:3.6.1-jdk-8-alpine \ + mvn -q test -Dcxcommon.version=${env.cxCommonVersion} -Dsast.version=9.3 -Dtest=com.cx.automation.plugin.test.cxcommonclient.scan.*,com.cx.automation.plugin.test.cxcommonclient.osa.* \ + -Dtopology.xml.ref=/usr/src/automation/env/topology.xml -Denv=hardening_env -DfailIfNoTests=false -Dmaven.test.failure.ignore=false -DskipTests=false" + } + } + } + } + post { + always { + script { + dir("${workspace}/9.3/Checkmarx-System-Test") { + junit '**/PluginsCommonClient/target/surefire-reports/**/*.xml' + } + if (ipAddress93 != null) { + node(vmName + "-9.3") { + kit.zipStashInstallationLogs("CxSAST-9.3-Logs") + } + unstash "CxSAST-9.3-Logs" + deleteVm(provider, ipAddress93, vmName + "-9.3") + } + } + } + failure { + script { + env.PIPELINE_STATUS = "Failure" + env.STAGE_NAME_FAILED="${STAGE_NAME}" + } + } + } + } + } + } + + stage('Publish') { + when { + expression { + BRANCH_NAME == 'master' + } + } + parallel { + + stage('GitHub Release') { + environment { + GITHUB_TOKEN = credentials('github-cxflowtestuser') + } + steps { + script { + sh "ghr -t ${GITHUB_TOKEN} -u checkmarx-ltd -n ${env.cxCommonVersion} -r Cx-Client-Common -c ${GIT_COMMIT} -delete ${env.cxCommonVersion} ./target/cx-client-common-${env.cxCommonVersion}.jar" + } + } + post { + failure { + script { + env.PIPELINE_STATUS = "Failure" + env.STAGE_NAME_FAILED="${STAGE_NAME}" + } + } + } + } + + stage('Maven Central') { + environment { + GNUPG_TOKEN = credentials('gnupg-credentials') + } + steps { + script { + //sh "docker run --rm --name publish-${BUILD_TAG} -v /root/.m2:/root/.m2 -v ${workspace}:/usr/src/cx-common -v /root/.gnupg:/root/.gnupg -w /usr/src/cx-common maven:3.6.1-jdk-8 mvn deploy -P release" + sh "mvn deploy -P release -DskipTests -Dcxcommon.version=${env.cxCommonVersion} -Dgpg.passphrase=${GNUPG_TOKEN}" + } + } + post { + failure { + script { + env.PIPELINE_STATUS = "Failure" + env.STAGE_NAME_FAILED="${STAGE_NAME}" + } + } + } + } + + stage('cx-artifactory') { + steps { + script { + kit.Upload_To_Artifactory("./target/cx-client-common-${env.cxCommonVersion}.jar", "libs-release-local/com/checkmarx/cx-client-common/${env.cxCommonVersion}/cx-client-common-${env.cxCommonVersion}.jar") + } + } + post { + failure { + script { + env.PIPELINE_STATUS = "Failure" + env.STAGE_NAME_FAILED="${STAGE_NAME}" + } + } + } + } + } + } + } + + post { + always { + archiveArtifacts "*.zip, target/cx-client-common-${env.cxCommonVersion}.jar" + script { + try { + kit.Command_Execution_Sh("jq -n env > env.json") + kit.Command_Execution_Sh("curl -sb -k -v -H \"Content-type: application/json\" -XPOST http://cx-elastic01:9200/cx-client-common/_doc -d @env.json") + } catch (Exception e) { + kit.Warning_Msg("The message couldn't be pushed to elastic, error:\n" + e.toString()) + } + } + } + cleanup { + cleanWs() + } + } +} diff --git a/README.md b/README.md new file mode 100644 index 00000000..bfcdda52 --- /dev/null +++ b/README.md @@ -0,0 +1,11 @@ +# cx-client-common +Client library that allows Java applications to interact with Checkmarx products. + +## Release Notes +Please read latest features and fixes from the [Releases](https://github.com/checkmarx-ltd/Cx-Client-Common/releases/latest). + +## Build +mvn clean install + +## Contributing +Please read through our [contributing guidelines](CONTRIBUTING.md). \ No newline at end of file diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 00000000..7ad3bb1f --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,40 @@ +# Maven +# Build your Java project and run tests with Apache Maven. +# Add steps that analyze code, save build artifacts, deploy, and more: +# https://docs.microsoft.com/azure/devops/pipelines/languages/java + +trigger: +- Version_9.00 + +pool: + vmImage: 'ubuntu-latest' + +steps: +- task: Maven@3 + inputs: + mavenPomFile: 'pom.xml' + mavenOptions: '-Xmx3072m' + javaHomeOption: 'JDKVersion' + jdkVersionOption: '1.8' + jdkArchitectureOption: 'x64' + publishJUnitResults: false + goals: 'package' +- task: S3Upload@1 + inputs: + awsCredentials: 'CxSLDC bucket' + regionName: 'eu-west-1' + bucketName: 'cxsdlc' + sourceFolder: 'target' + globExpressions: 'cx-client-common-*.jar' + targetFolder: '$(Build.BuildID)' + filesAcl: 'bucket-owner-full-control' + createBucket: true +- task: S3Upload@1 + inputs: + awsCredentials: 'CxSLDC bucket' + regionName: 'eu-west-1' + bucketName: 'cxsdlc' + sourceFolder: 'target' + globExpressions: 'cx-client-common-*.jar' + filesAcl: 'bucket-owner-full-control' + createBucket: true diff --git a/log4j.xml b/log4j.xml new file mode 100644 index 00000000..3d5609a9 --- /dev/null +++ b/log4j.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index 86282af6..f41ee4ee 100644 --- a/pom.xml +++ b/pom.xml @@ -1,30 +1,38 @@ - 4.0.0 com.checkmarx cx-client-common - 8.90.0-SNAPSHOT + ${cxcommon.version} jar - Checkmarx Client - Enables web services access Checkmarx SAST and OSA scan. + Web client for interaction with Checkmarx SAST, SCA and OSA products https://www.checkmarx.com - scm:git:git://github.com/CxRepositories/Cx-Client-Common.git - scm:git:ssh://github.com/CxRepositories/Cx-Client-Common.git - https://github.com/CxRepositories/Cx-Client-Common/tree/master + scm:git:git://github.com/checkmarx-ltd/Cx-Client-Common.git + scm:git:ssh://github.com/checkmarx-ltd/Cx-Client-Common.git + https://github.com/checkmarx-ltd/Cx-Client-Common/tree/master + MIT License http://www.opensource.org/licenses/mit-license.php - UTF-8 + 1.7.31 + 1.2.0 + 2.3.0 + 1.18.6 + + + checkmarx-ltd_Cx-Client-Common + checkmarx-ltd + https://sonarcloud.io + Cx-Client-Common Checkmarx @@ -59,14 +67,52 @@ maven-compiler-plugin 3.7.0 - 1.7 - 1.7 + 1.8 + 1.8 + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.3.0 + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + + + + src/main/resources + true + + - + + org.slf4j + slf4j-api + ${slf4j.version} + provided + + + junit + junit + 4.13.1 + test + org.freemarker freemarker @@ -75,22 +121,52 @@ org.apache.httpcomponents httpmime - 4.4.1 + 4.5.4 org.apache.httpcomponents httpcore - 4.4 + 4.4.12 + + + org.apache.httpcomponents + httpclient-win + 4.5.10 - com.fasterxml.jackson.core - jackson-databind - 2.8.9 + org.bouncycastle + bcprov-jdk15on + 1.68 - org.whitesource - whitesource-fs-agent - 18.7.1 + plexus-utils + org.codehaus.plexus + 3.2.0 + + + org.codehaus.plexus + plexus-archiver + 4.1.0 + + + commons-compress + org.apache.commons + + + plexus-utils + org.codehaus.plexus + + + + + com.google.guava + guava + 27.0-jre + + + com.checkmarx + cx-ws-fs-agent + 20.0.11 javax.xml.bind @@ -101,19 +177,89 @@ logback-classic - org.slf4j - slf4j-api + bcpg-jdk15on + org.bouncycastle + + + plexus-archiver + org.codehaus.plexus + + + commons-collections + commons-collections + + + guava + com.google.guava + + + plexus-utils + org.codehaus.plexus + + + + + com.beust + jcommander + + + org.apache.ant + ant + + + org.springframework + spring-web + - org.slf4j - slf4j-api - 1.7.5 + com.beust + jcommander + 1.78 + + + org.apache.ant + ant + 1.10.9 + + + org.springframework + spring-web + 5.3.5 + + + + org.projectlombok + lombok + ${lombok.version} provided - + + org.awaitility + awaitility + 4.0.2 + + + org.apache.commons + commons-lang3 + 3.10 + + + + + jakarta.xml.bind + jakarta.xml.bind-api + 2.3.2 + + + + org.glassfish.jaxb + jaxb-runtime + 2.3.2 + + ossrh @@ -124,7 +270,6 @@ https://oss.sonatype.org/service/local/staging/deploy/maven2/ - @@ -134,7 +279,6 @@ - release @@ -175,6 +319,7 @@ ossrh https://oss.sonatype.org/ true + 10 @@ -182,8 +327,7 @@ maven-gpg-plugin 1.5 - C:\Program Files (x86)\GNU\GnuPG\gpg2.exe - Checkmarx123456 + /usr/bin/gpg @@ -199,71 +343,12 @@ - - Gal Or Nussbaum - gal.nussbaum@checkmarx.com - Checkmarx - https://www.checkmarx.com/ - - Architect - Developer - - - https://www.linkedin.com/in/gal-nussbaum-68b3a76a/de - - - - Dor Golan - dor.golan@checkmarx.com - Checkmarx - https://www.checkmarx.com/ - - Architect - Developer - - - http://i.imgur.com/44Iil53.png - - - - Shaul Valero - shaul.valero@checkmarx.com - Checkmarx - https://www.checkmarx.com/ - - Dev TL - Developer - - - http://i.imgur.com/44Iil53.png - - - - Eyal Amor - eyal.amor@checkmarx.com - Checkmarx - https://www.checkmarx.com/ - - Architect - Developer - - - http://i.imgur.com/44Iil53.png - - - - Yair David - Yair.David@checkmarx.com - Checkmarx + Alexey Kononov + alexey.kononov@checkmarx.com + Checkmarx Ltd https://www.checkmarx.com/ - - QA Manager - - - http://i.imgur.com/EVIS8LO.jpg - - \ No newline at end of file + diff --git a/src/main/java/com/cx/restclient/CxClientDelegator.java b/src/main/java/com/cx/restclient/CxClientDelegator.java new file mode 100644 index 00000000..d00de016 --- /dev/null +++ b/src/main/java/com/cx/restclient/CxClientDelegator.java @@ -0,0 +1,203 @@ +package com.cx.restclient; + +import com.cx.restclient.ast.AstSastClient; +import com.cx.restclient.ast.AstScaClient; +import com.cx.restclient.ast.dto.sca.AstScaResults; +import com.cx.restclient.common.Scanner; +import com.cx.restclient.common.summary.SummaryUtils; +import com.cx.restclient.configuration.CxScanConfig; +import com.cx.restclient.configuration.PropertyFileLoader; +import com.cx.restclient.dto.Results; +import com.cx.restclient.dto.ScanResults; +import com.cx.restclient.dto.ScannerType; +import com.cx.restclient.osa.dto.OSAResults; +import com.cx.restclient.sast.dto.SASTResults; +import com.cx.restclient.sast.utils.State; +import org.slf4j.Logger; + +import java.net.MalformedURLException; +import java.util.EnumMap; +import java.util.Map; + +import static com.cx.restclient.common.CxPARAM.PROJECT_POLICY_COMPLIANT_STATUS; +import static com.cx.restclient.common.CxPARAM.PROJECT_POLICY_VIOLATED_STATUS; +import static com.cx.restclient.cxArm.utils.CxARMUtils.getPoliciesNames; + +/** + * Created by Galn on 05/02/2018. + */ + +public class CxClientDelegator implements Scanner { + private static final PropertyFileLoader properties = PropertyFileLoader.getDefaultInstance(); + + private static final String PRINT_LINE = "-----------------------------------------------------------------------------------------"; + + private Logger log; + private CxScanConfig config; + + Map scannersMap = new EnumMap<>(ScannerType.class); + + public CxClientDelegator(CxScanConfig config, Logger log) throws MalformedURLException { + + this.config = config; + this.log = log; + if (config.isAstSastEnabled()) { + scannersMap.put(ScannerType.AST_SAST, new AstSastClient(config, log)); + } + + if (config.isSastEnabled()) { + scannersMap.put(ScannerType.SAST, new CxSASTClient(config, log)); + } + + if (config.isOsaEnabled()) { + scannersMap.put(ScannerType.OSA, new CxOSAClient(config, log)); + } + + if (config.isAstScaEnabled()) { + scannersMap.put(ScannerType.AST_SCA, new AstScaClient(config, log)); + } + } + + + public CxClientDelegator(String serverUrl, String username, String password, String origin, boolean disableCertificateValidation, Logger log) throws MalformedURLException { + this(new CxScanConfig(serverUrl, username, password, origin, disableCertificateValidation), log); + } + + @Override + public ScanResults init() { + log.info("Initializing Cx client [{}]", properties.get("version")); + ScanResults scanResultsCombined = new ScanResults(); + + scannersMap.forEach((key, scanner) -> { + Results scanResults = scanner.init(); + scanResultsCombined.put(key, scanResults); + }); + + return scanResultsCombined; + } + + + @Override + public ScanResults initiateScan() { + + ScanResults scanResultsCombined = new ScanResults(); + + scannersMap.forEach((key, scanner) -> { + if (scanner.getState() == State.SUCCESS) { + Results scanResults = scanner.initiateScan(); + scanResultsCombined.put(key, scanResults); + } + }); + + return scanResultsCombined; + } + + + @Override + public ScanResults waitForScanResults() { + + ScanResults scanResultsCombined = new ScanResults(); + + scannersMap.forEach((key, scanner) -> { + if (scanner.getState() == State.SUCCESS) { + Results scanResults = scanner.waitForScanResults(); + scanResultsCombined.put(key, scanResults); + } + }); + + return scanResultsCombined; + } + + @Override + public ScanResults getLatestScanResults() { + + ScanResults scanResultsCombined = new ScanResults(); + + scannersMap.forEach((key, scanner) -> { + if (scanner.getState() == State.SUCCESS) { + Results scanResults = scanner.getLatestScanResults(); + scanResultsCombined.put(key, scanResults); + } + }); + + return scanResultsCombined; + + } + + public void printIsProjectViolated(ScanResults scanResults) { + if (config.getEnablePolicyViolations()) { + log.info(PRINT_LINE); + log.info("Policy Management: "); + log.info("--------------------"); + + OSAResults osaResults = (OSAResults) scanResults.get(ScannerType.OSA); + SASTResults sastResults = (SASTResults) scanResults.get(ScannerType.SAST); + AstScaResults scaResults = (AstScaResults) scanResults.get(ScannerType.AST_SCA); + + boolean hasOsaViolations = + osaResults != null && + osaResults.getOsaPolicies() != null && + !osaResults.getOsaPolicies().isEmpty(); + + boolean hasSastPolicies = false; + + if (sastResults != null && sastResults.getSastPolicies() != null && !sastResults.getSastPolicies().isEmpty()) { + hasSastPolicies = true; + } + + boolean hasScaViolations = false; + if (scaResults != null && scaResults.isPolicyViolated()) { + hasScaViolations = true; + } + + if (!hasSastPolicies && !hasOsaViolations && !hasScaViolations) { + log.info(PROJECT_POLICY_COMPLIANT_STATUS); + log.info(PRINT_LINE); + } else { + log.info(PROJECT_POLICY_VIOLATED_STATUS); + if (hasSastPolicies) { + log.info("SAST violated policies names: {}", getPoliciesNames(sastResults.getSastPolicies())); + } + if (hasOsaViolations) { + log.info("OSA violated policies names: {}", getPoliciesNames(osaResults.getOsaPolicies())); + } + if (hasScaViolations) { + log.info("SCA policies are violated."); + } + log.info(PRINT_LINE); + } + + } + } + + + public String generateHTMLSummary(ScanResults combinedResults) throws Exception { + + return SummaryUtils.generateSummary( + (SASTResults) combinedResults.get(ScannerType.SAST), + (OSAResults) combinedResults.get(ScannerType.OSA), + (AstScaResults) combinedResults.get(ScannerType.AST_SCA), config); + } + + public String generateHTMLSummary(SASTResults sastResults, OSAResults osaResults, AstScaResults scaResults) throws Exception { + return SummaryUtils.generateSummary(sastResults, osaResults, scaResults, config); + } + + public CxSASTClient getSastClient() { + return (CxSASTClient) scannersMap.get(ScannerType.SAST); + } + + public CxOSAClient getOsaClient() { + return (CxOSAClient) scannersMap.get(ScannerType.OSA); + } + + public AstScaClient getScaClient() { + return (AstScaClient) scannersMap.get(ScannerType.AST_SCA); + } + + public void close() { + scannersMap.values().forEach(Scanner::close); + } + + +} \ No newline at end of file diff --git a/src/main/java/com/cx/restclient/CxOSAClient.java b/src/main/java/com/cx/restclient/CxOSAClient.java index 8366665e..19815299 100644 --- a/src/main/java/com/cx/restclient/CxOSAClient.java +++ b/src/main/java/com/cx/restclient/CxOSAClient.java @@ -1,23 +1,30 @@ package com.cx.restclient; +import com.cx.restclient.common.Scanner; +import com.cx.restclient.common.ShragaUtils; import com.cx.restclient.common.Waiter; import com.cx.restclient.configuration.CxScanConfig; -import com.cx.restclient.cxArm.dto.Policy; -import com.cx.restclient.dto.Status; +import com.cx.restclient.dto.*; import com.cx.restclient.exception.CxClientException; import com.cx.restclient.httpClient.CxHttpClient; import com.cx.restclient.osa.dto.*; import com.cx.restclient.osa.utils.OSAUtils; +import com.cx.restclient.sast.utils.LegacyClient; +import com.cx.restclient.sast.utils.State; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.http.entity.StringEntity; import org.slf4j.Logger; import org.whitesource.fs.ComponentScan; import java.io.IOException; +import java.net.MalformedURLException; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Properties; import static com.cx.restclient.cxArm.dto.CxProviders.OPEN_SOURCE; -import static com.cx.restclient.cxArm.utils.CxARMUtils.getProjectViolations; +import static com.cx.restclient.cxArm.utils.CxARMUtils.getProjectViolatedPolicies; import static com.cx.restclient.httpClient.utils.ContentType.CONTENT_TYPE_APPLICATION_JSON_V1; import static com.cx.restclient.httpClient.utils.HttpClientHelper.convertToJson; import static com.cx.restclient.osa.utils.OSAParam.*; @@ -26,36 +33,65 @@ /** * Created by Galn on 05/02/2018. */ -class CxOSAClient { - - private CxHttpClient httpClient; - private Logger log; - private CxScanConfig config; - private Waiter osaWaiter = new Waiter("CxOSA scan", 20) { - @Override - public OSAScanStatus getStatus(String id) throws CxClientException, IOException { - return getOSAScanStatus(id); - } +public class CxOSAClient extends LegacyClient implements Scanner { - @Override - public void printProgress(OSAScanStatus scanStatus) { - printOSAProgress(scanStatus, getStartTimeSec()); - } + private Waiter osaWaiter; + + private String scanId; + private OSAResults osaResults = new OSAResults(); - @Override - public OSAScanStatus resolveStatus(OSAScanStatus scanStatus) throws CxClientException { - return resolveOSAStatus(scanStatus); - } - }; - public CxOSAClient(CxHttpClient client, Logger log, CxScanConfig config) { - this.log = log; - this.httpClient = client; - this.config = config; + public OSAScanStatus getStatus(String id) throws IOException { + return getOSAScanStatus(id); } - //API - public String createOSAScan(long projectId) throws IOException, CxClientException { + public CxOSAClient(CxScanConfig config, Logger log) throws MalformedURLException { + super(config, log); + int interval = config.getOsaProgressInterval() != null ? config.getOsaProgressInterval() : 20; + int retry = config.getConnectionRetries() != null ? config.getConnectionRetries() : 3; + osaWaiter = new Waiter("CxOSA scan", interval, retry) { + @Override + public OSAScanStatus getStatus(String id) throws IOException { + return getOSAScanStatus(id); + } + + @Override + public void printProgress(OSAScanStatus scanStatus) { + printOSAProgress(scanStatus, getStartTimeSec()); + } + + @Override + public OSAScanStatus resolveStatus(OSAScanStatus scanStatus) { + return resolveOSAStatus(scanStatus); + } + }; + } + + @Override + public Results init() { + OSAResults initOsaResults = new OSAResults(); + try { + initiate(); + } catch (CxClientException e) { + log.error(e.getMessage()); + setState(State.FAILED); + initOsaResults.setException(e); + } + return initOsaResults; + } + + @Override + public Results initiateScan() { + osaResults = new OSAResults(); + try { + ensureProjectIdSpecified(); + } catch (CxClientException e) { + log.error(e.getMessage()); + setState(State.FAILED); + osaResults.setException(e); + return osaResults; + } + log.info("----------------------------------- Create CxOSA Scan:------------------------------------"); log.info("Creating OSA scan"); String osaDependenciesJson = config.getOsaDependenciesJson(); @@ -63,13 +99,34 @@ public String createOSAScan(long projectId) throws IOException, CxClientExceptio try { osaDependenciesJson = resolveOSADependencies(); } catch (Exception e) { - throw new CxClientException("Failed to resolve dependencies for OSA scan: " + e.getMessage(), e); + log.error(e.getMessage()); + setState(State.FAILED); + osaResults.setException(new CxClientException("Failed to resolve dependencies for OSA scan: " + e.getMessage(), e)); + return osaResults; } } - return sendOSAScan(osaDependenciesJson, projectId); + + try { + scanId = sendOSAScan(osaDependenciesJson, projectId); + } catch (IOException e) { + scanId = null; + log.error(e.getMessage()); + setState(State.FAILED); + osaResults.setException(new CxClientException("Error sending OSA scan request.", e)); + return osaResults; + } + + osaResults.setOsaProjectSummaryLink(config.getUrl(), projectId); + osaResults.setOsaScanId(scanId); + return osaResults; } - private String resolveOSADependencies() { + + public void setOsaFSAProperties(Properties fsaConfig) { //For CxMaven plugin + config.setOsaFsaConfig(fsaConfig); + } + + private String resolveOSADependencies() throws JsonProcessingException { log.info("Scanning for CxOSA compatible files"); Properties scannerProperties = config.getOsaFsaConfig(); if (scannerProperties == null) { @@ -77,42 +134,64 @@ private String resolveOSADependencies() { config.getOsaFolderExclusions(), config.getOsaFilterPattern(), config.getOsaArchiveIncludePatterns(), - config.getSourceDir(), + config.getEffectiveSourceDirForDependencyScan(), config.getOsaRunInstall(), + config.getOsaScanDepth(), log); } + ObjectMapper mapper = new ObjectMapper(); + log.info("Scanner properties: " + mapper.writerWithDefaultPrettyPrinter().writeValueAsString(scannerProperties.toString())); ComponentScan componentScan = new ComponentScan(scannerProperties); String osaDependenciesJson = componentScan.scan(); - OSAUtils.writeToOsaListToFile(config.getReportsDir(), osaDependenciesJson, log); - + OSAUtils.writeToOsaListToFile(OSAUtils.getWorkDirectory(config.getReportsDir(), config.getOsaGenerateJsonReport()), osaDependenciesJson, log); return osaDependenciesJson; } - public OSAResults getOSAResults(String scanId, long projectId) throws CxClientException, InterruptedException, IOException { - log.info("-------------------------------------Get CxOSA Results:-----------------------------------"); - log.info("Waiting for OSA scan to finish"); - OSAScanStatus osaScanStatus = osaWaiter.waitForTaskToFinish(scanId, -1, log); - log.info("OSA scan finished successfully. Retrieving OSA scan results"); + @Override + public CxHttpClient getHttpClient() { + return httpClient; + } - log.info("Creating OSA reports"); - OSAResults osaResults = retrieveOSAResults(scanId, osaScanStatus, projectId); + @Override + public Results waitForScanResults() { + try { + ensureProjectIdSpecified(); - if (config.getEnablePolicyViolations()) { - resolveOSAViolation(osaResults, projectId); - } + if (scanId == null) { + throw new CxClientException("Scan was not created."); + } + + log.info("-------------------------------------Get CxOSA Results:-----------------------------------"); + log.info("Waiting for OSA scan to finish"); + + OSAScanStatus osaScanStatus; + osaScanStatus = osaWaiter.waitForTaskToFinish(scanId, this.config.getOsaScanTimeoutInMinutes(), log); + log.info("OSA scan finished successfully. Retrieving OSA scan results"); + + log.info("Creating OSA reports"); + + osaResults = retrieveOSAResults(scanId, osaScanStatus, projectId); + + if (config.getEnablePolicyViolations()) { + resolveOSAViolation(osaResults, projectId); + } - OSAUtils.printOSAResultsToConsole(osaResults, config.getEnablePolicyViolations(), log); + OSAUtils.printOSAResultsToConsole(osaResults, config.getEnablePolicyViolations(), log); - if (config.getReportsDir() != null) { - writeJsonToFile(OSA_SUMMARY_NAME, osaResults.getResults(), config.getReportsDir(), log); - writeJsonToFile(OSA_LIBRARIES_NAME, osaResults.getOsaLibraries(), config.getReportsDir(), log); - writeJsonToFile(OSA_VULNERABILITIES_NAME, osaResults.getOsaVulnerabilities(), config.getReportsDir(), log); + if (config.getReportsDir() != null) { + writeJsonToFile(OSA_SUMMARY_NAME, osaResults.getResults(), config.getReportsDir(), config.getOsaGenerateJsonReport(), log); + writeJsonToFile(OSA_LIBRARIES_NAME, osaResults.getOsaLibraries(), config.getReportsDir(), config.getOsaGenerateJsonReport(), log); + writeJsonToFile(OSA_VULNERABILITIES_NAME, osaResults.getOsaVulnerabilities(), config.getReportsDir(), config.getOsaGenerateJsonReport(), log); + } + } catch (Exception e) { + log.error(e.getMessage()); + osaResults.setException(new CxClientException("Failed to retrieve OSA results.", e)); } return osaResults; } - private OSAResults retrieveOSAResults(String scanId, OSAScanStatus osaScanStatus, long projectId) throws CxClientException, IOException { + private OSAResults retrieveOSAResults(String scanId, OSAScanStatus osaScanStatus, long projectId) throws IOException { OSASummaryResults osaSummaryResults = getOSAScanSummaryResults(scanId); List osaLibraries = getOSALibraries(scanId); List osaVulnerabilities = getOSAVulnerabilities(scanId); @@ -122,31 +201,40 @@ private OSAResults retrieveOSAResults(String scanId, OSAScanStatus osaScanStatus return results; } - private void resolveOSAViolation(OSAResults osaResults, long projectId){ + private void resolveOSAViolation(OSAResults osaResults, long projectId) { try { - for (Policy policy : getProjectViolations(httpClient, config.getCxARMUrl(), projectId, OPEN_SOURCE.value())) { - osaResults.getOsaPolicies().add(policy.getPolicyName()); - osaResults.addAllViolations(policy.getViolations()); - } - }catch (Exception ex) { - log.error("CxARM is not available. Policy violations for OSA cannot be calculated: " + ex.getMessage()); + getProjectViolatedPolicies(httpClient, config.getCxARMUrl(), projectId, OPEN_SOURCE.value()) + .forEach(osaResults::addPolicy); + } catch (Exception ex) { + throw new CxClientException("CxARM is not available. Policy violations for OSA cannot be calculated: " + ex.getMessage()); } } + @Override + public Results getLatestScanResults() { + osaResults = new OSAResults(); + try { + ensureProjectIdSpecified(); + + log.info("----------------------------------Get CxOSA Last Results:--------------------------------"); - public OSAResults getLatestOSAResults(long projectId) throws CxClientException, IOException, InterruptedException { - log.info("----------------------------------Get CxOSA Last Results:--------------------------------"); - List scanList = getOSALastOSAStatus(projectId); - for (OSAScanStatus s : scanList) { - if (Status.SUCCEEDED.value().equals(s.getState().getName())) { - return retrieveOSAResults(s.getId(), s, projectId); + List scanList = getOSALastOSAStatus(projectId); + for (OSAScanStatus s : scanList) { + if (Status.SUCCEEDED.value().equals(s.getState().getName())) { + osaResults = retrieveOSAResults(s.getId(), s, projectId); + break; + } } + } catch (Exception e) { + log.error(e.getMessage()); + osaResults.setException(new CxClientException("Error getting last scan results.", e)); } - return new OSAResults(); + + return osaResults; } //Private Methods - private String sendOSAScan(String osaDependenciesJson, long projectId) throws CxClientException, IOException { + private String sendOSAScan(String osaDependenciesJson, long projectId) throws IOException { log.info("Sending OSA scan request"); CreateOSAScanResponse osaScan = sendOSARequest(projectId, osaDependenciesJson); String summaryLink = OSAUtils.composeProjectOSASummaryLink(config.getUrl(), projectId); @@ -155,33 +243,33 @@ private String sendOSAScan(String osaDependenciesJson, long projectId) throws Cx return osaScan.getScanId(); } - private CreateOSAScanResponse sendOSARequest(long projectId, String osaDependenciesJson) throws IOException, CxClientException { + private CreateOSAScanResponse sendOSARequest(long projectId, String osaDependenciesJson) throws IOException { CreateOSAScanRequest req = new CreateOSAScanRequest(projectId, osaDependenciesJson); - StringEntity entity = new StringEntity(convertToJson(req)); + StringEntity entity = new StringEntity(convertToJson(req), StandardCharsets.UTF_8); return httpClient.postRequest(OSA_SCAN_PROJECT, CONTENT_TYPE_APPLICATION_JSON_V1, entity, CreateOSAScanResponse.class, 201, "create OSA scan"); } - private OSASummaryResults getOSAScanSummaryResults(String scanId) throws IOException, CxClientException { + private OSASummaryResults getOSAScanSummaryResults(String scanId) throws IOException { String relativePath = OSA_SCAN_SUMMARY + SCAN_ID_QUERY_PARAM + scanId; return httpClient.getRequest(relativePath, CONTENT_TYPE_APPLICATION_JSON_V1, OSASummaryResults.class, 200, "OSA scan summary results", false); } - private List getOSALastOSAStatus(long projectId) throws IOException, CxClientException { + private List getOSALastOSAStatus(long projectId) throws IOException { return (List) httpClient.getRequest(OSA_SCANS + PROJECT_ID_QUERY_PARAM + projectId, CONTENT_TYPE_APPLICATION_JSON_V1, OSAScanStatus.class, 200, " last OSA scan ID", true); } - private List getOSALibraries(String scanId) throws IOException, CxClientException { + private List getOSALibraries(String scanId) throws IOException { String relPath = OSA_SCAN_LIBRARIES + SCAN_ID_QUERY_PARAM + scanId + ITEM_PER_PAGE_QUERY_PARAM + MAX_ITEMS; return (List) httpClient.getRequest(relPath, CONTENT_TYPE_APPLICATION_JSON_V1, Library.class, 200, "OSA libraries", true); } - private List getOSAVulnerabilities(String scanId) throws CxClientException, IOException { + private List getOSAVulnerabilities(String scanId) throws IOException { String relPath = OSA_SCAN_VULNERABILITIES + SCAN_ID_QUERY_PARAM + scanId + ITEM_PER_PAGE_QUERY_PARAM + MAX_ITEMS; return (List) httpClient.getRequest(relPath, CONTENT_TYPE_APPLICATION_JSON_V1, CVE.class, 200, "OSA vulnerabilities", true); } //Waiter - overload methods - private OSAScanStatus getOSAScanStatus(String scanId) throws CxClientException, IOException { + private OSAScanStatus getOSAScanStatus(String scanId) throws IOException { String relPath = OSA_SCAN_STATUS.replace("{scanId}", scanId); OSAScanStatus scanStatus = httpClient.getRequest(relPath, CONTENT_TYPE_APPLICATION_JSON_V1, OSAScanStatus.class, 200, "OSA scan status", false); int stateId = scanStatus.getState().getId(); @@ -197,27 +285,31 @@ private OSAScanStatus getOSAScanStatus(String scanId) throws CxClientException, } private void printOSAProgress(OSAScanStatus scanStatus, long startTime) { - long elapsedSec = System.currentTimeMillis() / 1000 - startTime; - long hours = elapsedSec / 3600; - long minutes = elapsedSec % 3600 / 60; - long seconds = elapsedSec % 60; - String hoursStr = (hours < 10) ? ("0" + Long.toString(hours)) : (Long.toString(hours)); - String minutesStr = (minutes < 10) ? ("0" + Long.toString(minutes)) : (Long.toString(minutes)); - String secondsStr = (seconds < 10) ? ("0" + Long.toString(seconds)) : (Long.toString(seconds)); - log.info("Waiting for OSA scan results. Elapsed time: " + hoursStr + ":" + minutesStr + ":" + secondsStr + ". " + + String timestamp = ShragaUtils.getTimestampSince(startTime); + + log.info("Waiting for OSA scan results. Elapsed time: " + timestamp + ". " + "Status: " + scanStatus.getState().getName()); } - private OSAScanStatus resolveOSAStatus(OSAScanStatus scanStatus) throws CxClientException { - if (Status.FAILED == scanStatus.getBaseStatus()) { + private OSAScanStatus resolveOSAStatus(OSAScanStatus scanStatus) { + if (scanStatus == null) { + throw new CxClientException("OSA scan cannot be completed."); + } else if (Status.FAILED == scanStatus.getBaseStatus()) { String failedMsg = scanStatus.getState() == null ? "" : "status [" + scanStatus.getState().getName() + "]. Reason: " + scanStatus.getState().getFailureReason(); throw new CxClientException("OSA scan cannot be completed. " + failedMsg); } if (Status.SUCCEEDED == scanStatus.getBaseStatus()) { log.info("OSA scan finished."); - return scanStatus; } return scanStatus; } + + private void ensureProjectIdSpecified() { + if (projectId == 0) { + throw new CxClientException("projectId must be set before executing this method."); + } + } + + } diff --git a/src/main/java/com/cx/restclient/CxSASTClient.java b/src/main/java/com/cx/restclient/CxSASTClient.java index 9cc88e00..4bbff3c6 100644 --- a/src/main/java/com/cx/restclient/CxSASTClient.java +++ b/src/main/java/com/cx/restclient/CxSASTClient.java @@ -1,29 +1,38 @@ package com.cx.restclient; +import com.cx.restclient.common.Scanner; +import com.cx.restclient.common.ShragaUtils; import com.cx.restclient.common.Waiter; import com.cx.restclient.configuration.CxScanConfig; -import com.cx.restclient.cxArm.dto.Policy; -import com.cx.restclient.dto.Status; +import com.cx.restclient.dto.*; import com.cx.restclient.exception.CxClientException; -import com.cx.restclient.httpClient.CxHttpClient; import com.cx.restclient.sast.dto.*; +import com.cx.restclient.sast.utils.LegacyClient; import com.cx.restclient.sast.utils.SASTUtils; +import com.cx.restclient.sast.utils.State; import com.cx.restclient.sast.utils.zip.CxZipUtils; +import com.google.gson.Gson; import org.apache.http.HttpEntity; +import org.apache.http.entity.BufferedHttpEntity; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.entity.mime.HttpMultipartMode; import org.apache.http.entity.mime.MultipartEntityBuilder; import org.apache.http.entity.mime.content.InputStreamBody; +import org.json.JSONObject; import org.slf4j.Logger; -import java.io.File; -import java.io.FileInputStream; +import java.io.ByteArrayInputStream; import java.io.IOException; -import java.util.List; +import java.io.InputStream; +import java.net.MalformedURLException; +import java.nio.charset.StandardCharsets; +import java.text.SimpleDateFormat; +import java.util.*; +import java.util.stream.Collectors; import static com.cx.restclient.cxArm.dto.CxProviders.SAST; -import static com.cx.restclient.cxArm.utils.CxARMUtils.getProjectViolations; +import static com.cx.restclient.cxArm.utils.CxARMUtils.getProjectViolatedPolicies; import static com.cx.restclient.httpClient.utils.ContentType.*; import static com.cx.restclient.httpClient.utils.HttpClientHelper.convertToJson; import static com.cx.restclient.sast.utils.SASTParam.*; @@ -33,61 +42,276 @@ /** * Created by Galn on 05/02/2018. */ - -class CxSASTClient { - private Logger log; - private CxHttpClient httpClient; - private CxScanConfig config; - private int reportTimeoutSec = 500; - private Waiter sastWaiter = new Waiter("CxSAST scan", 20) { +public class CxSASTClient extends LegacyClient implements Scanner { + + public static final String JENKINS = "jenkins"; + + private int reportTimeoutSec = 5000; + private int cxARMTimeoutSec = 1000; + private Waiter sastWaiter; + private static final String SCAN_ID_PATH_PARAM = "{scanId}"; + private static final String PROJECT_ID_PATH_PARAM = "{projectId}"; + private static final String SCAN_WITH_SETTINGS_URL = "sast/scanWithSettings"; + private static final String ENGINE_CONFIGURATION_ID_DEFAULT = "0"; + private long scanId; + private SASTResults sastResults = new SASTResults(); + private static final String SWAGGER_LOCATION = "help/swagger/docs/v1.1"; + private static final String ZIPPED_SOURCE = "zippedSource"; + private Waiter reportWaiter = new Waiter("Scan report", 10, 3) { @Override - public ResponseQueueScanStatus getStatus(String id) throws CxClientException, IOException { - return getSASTScanStatus(id); + public ReportStatus getStatus(String id) throws IOException { + return getReportStatus(id); } @Override - public void printProgress(ResponseQueueScanStatus scanStatus) { - printSASTProgress(scanStatus, getStartTimeSec()); + public void printProgress(ReportStatus reportStatus) { + printReportProgress(reportStatus, getStartTimeSec()); } @Override - public ResponseQueueScanStatus resolveStatus(ResponseQueueScanStatus scanStatus) throws CxClientException { - return resolveSASTStatus(scanStatus); + public ReportStatus resolveStatus(ReportStatus reportStatus) { + return resolveReportStatus(reportStatus); + } + + //Report Waiter - overload methods + private ReportStatus getReportStatus(String reportId) throws CxClientException, IOException { + ReportStatus reportStatus = httpClient.getRequest(SAST_GET_REPORT_STATUS.replace("{reportId}", reportId), CONTENT_TYPE_APPLICATION_JSON_V1, ReportStatus.class, 200, " report status", false); + reportStatus.setBaseId(reportId); + String currentStatus = reportStatus.getStatus().getValue(); + if (currentStatus.equals(ReportStatusEnum.INPROCESS.value())) { + reportStatus.setBaseStatus(Status.IN_PROGRESS); + } else if (currentStatus.equals(ReportStatusEnum.FAILED.value())) { + reportStatus.setBaseStatus(Status.FAILED); + } else { + reportStatus.setBaseStatus(Status.SUCCEEDED); //todo fix it!! + } + + return reportStatus; + } + + private ReportStatus resolveReportStatus(ReportStatus reportStatus) throws CxClientException { + if (reportStatus != null) { + if (Status.SUCCEEDED == reportStatus.getBaseStatus()) { + return reportStatus; + } else { + throw new CxClientException("Generation of scan report [id=" + reportStatus.getBaseId() + "] failed."); + } + } else { + throw new CxClientException("Generation of scan report failed."); + } + } + + private void printReportProgress(ReportStatus reportStatus, long startTime) { + String reportType = reportStatus.getContentType().replace("application/", ""); + log.info("Waiting for server to generate " + reportType + " report. " + (startTime + reportTimeoutSec - (System.currentTimeMillis() / 1000)) + " seconds left to timeout"); } + }; - private Waiter reportWaiter = new Waiter("Scan report", 5) { + private Waiter cxARMWaiter = new Waiter("CxARM policy violations", 20, 3) { @Override - public ReportStatus getStatus(String id) throws CxClientException, IOException { - return getReportStatus(id); + public CxARMStatus getStatus(String id) throws IOException { + return getCxARMStatus(id); } @Override - public void printProgress(ReportStatus reportStatus) { - printReportProgress(reportStatus, getStartTimeSec()); + public void printProgress(CxARMStatus cxARMStatus) { + printCxARMProgress(getStartTimeSec()); } @Override - public ReportStatus resolveStatus(ReportStatus reportStatus) throws CxClientException { - return resolveReportStatus(reportStatus); + public CxARMStatus resolveStatus(CxARMStatus cxARMStatus) { + return resolveCxARMStatus(cxARMStatus); + } + + + //CxARM Waiter - overload methods + private CxARMStatus getCxARMStatus(String projectId) throws CxClientException, IOException { + CxARMStatus cxARMStatus = httpClient.getRequest(SAST_GET_CXARM_STATUS.replace(PROJECT_ID_PATH_PARAM, projectId), CONTENT_TYPE_APPLICATION_JSON_V1, CxARMStatus.class, 200, " cxARM status", false); + cxARMStatus.setBaseId(projectId); + + String currentStatus = cxARMStatus.getStatus(); + if (currentStatus.equals(CxARMStatusEnum.IN_PROGRESS.value())) { + cxARMStatus.setBaseStatus(Status.IN_PROGRESS); + } else if (currentStatus.equals(CxARMStatusEnum.FAILED.value())) { + cxARMStatus.setBaseStatus(Status.FAILED); + } else if (currentStatus.equals(CxARMStatusEnum.FINISHED.value())) { + cxARMStatus.setBaseStatus(Status.SUCCEEDED); + } else { + cxARMStatus.setBaseStatus(Status.FAILED); + } + + return cxARMStatus; + } + + private void printCxARMProgress(long startTime) { + log.info("Waiting for server to retrieve policy violations. " + (startTime + cxARMTimeoutSec - (System.currentTimeMillis() / 1000)) + " seconds left to timeout"); + } + + private CxARMStatus resolveCxARMStatus(CxARMStatus cxARMStatus) throws CxClientException { + if (cxARMStatus != null) { + if (Status.SUCCEEDED == cxARMStatus.getBaseStatus()) { + return cxARMStatus; + } else { + throw new CxClientException("Getting policy violations of project [id=" + cxARMStatus.getBaseId() + "] failed."); + } + } else { + throw new CxClientException("Getting policy violations of project failed."); + } } }; - CxSASTClient(CxHttpClient client, Logger log, CxScanConfig config) { - this.log = log; - this.httpClient = client; - this.config = config; + + CxSASTClient(CxScanConfig config, Logger log) throws MalformedURLException { + super(config, log); + + int interval = config.getProgressInterval() != null ? config.getProgressInterval() : 20; + int retry = config.getConnectionRetries() != null ? config.getConnectionRetries() : 3; + sastWaiter = new Waiter("CxSAST scan", interval, retry) { + @Override + public ResponseQueueScanStatus getStatus(String id) throws IOException { + return getSASTScanStatus(id); + } + + @Override + public void printProgress(ResponseQueueScanStatus scanStatus) { + printSASTProgress(scanStatus, getStartTimeSec()); + } + + @Override + public ResponseQueueScanStatus resolveStatus(ResponseQueueScanStatus scanStatus) { + return resolveSASTStatus(scanStatus); + } + }; + } + + @Override + public Results init() { + SASTResults initSastResults = new SASTResults(); + try { + initiate(); + } catch (CxClientException e) { + log.error(e.getMessage()); + setState(State.FAILED); + initSastResults.setException(e); + } + return initSastResults; } //**------ API ------**// //CREATE SAST scan - long createSASTScan(long projectId) throws IOException, CxClientException { - log.info("-----------------------------------Create CxSAST Scan:------------------------------------"); - ScanSettingResponse scanSettingResponse = getScanSetting(projectId); + private void createSASTScan(long projectId) { + try { + log.info("-----------------------------------Create CxSAST Scan:------------------------------------"); + if (config.isAvoidDuplicateProjectScans() != null && config.isAvoidDuplicateProjectScans() && projectHasQueuedScans(projectId)) { + throw new CxClientException("\nAvoid duplicate project scans in queue\n"); + } + if (config.getRemoteType() == null) { //scan is local + scanId = createLocalSASTScan(projectId); + } else { + scanId = createRemoteSourceScan(projectId); + } + sastResults.setScanId(scanId); + log.info("SAST scan created successfully: Scan ID is " + scanId); + sastResults.setSastScanLink(config.getUrl(), scanId, projectId); + } catch (Exception e) { + log.error(e.getMessage()); + setState(State.FAILED); + sastResults.setException(new CxClientException(e)); + } + } + + private long createLocalSASTScan(long projectId) throws IOException { + if(isScanWithSettingsSupported()){ + log.info("Uploading the zipped source code."); + PathFilter filter = new PathFilter(config.getSastFolderExclusions(), config.getSastFilterPattern(), log); + byte[] zipFile = CxZipUtils.getZippedSources(config, filter, config.getSourceDir(), log); + ScanWithSettingsResponse response = scanWithSettings(zipFile,projectId,false ); + return response.getId(); + }else{ + configureScanSettings(projectId); + //prepare sources for scan + PathFilter filter = new PathFilter(config.getSastFolderExclusions(), config.getSastFilterPattern(), log); + byte[] zipFile = CxZipUtils.getZippedSources(config, filter, config.getSourceDir(), log); + uploadZipFile(zipFile, projectId); + + return createScan(projectId); + } + } + + private long createRemoteSourceScan(long projectId) throws IOException { + HttpEntity entity; + excludeProjectSettings(projectId); + RemoteSourceRequest req = new RemoteSourceRequest(config); + RemoteSourceTypes type = req.getType(); + boolean isSSH = false; + + switch (type) { + case SVN: + if (req.getPrivateKey() != null && req.getPrivateKey().length > 1) { + isSSH = true; + MultipartEntityBuilder builder = MultipartEntityBuilder.create(); + builder.addBinaryBody("privateKey", req.getPrivateKey(), ContentType.APPLICATION_JSON, null) + .addTextBody("absoluteUrl", req.getUri().getAbsoluteUrl()) + .addTextBody("port", String.valueOf(req.getUri().getPort())) + .addTextBody("paths", config.getSourceDir()); //todo add paths to req OR using without + entity = builder.build(); + } else { + entity = new StringEntity(convertToJson(req), ContentType.APPLICATION_JSON); + } + break; + case TFS: + entity = new StringEntity(convertToJson(req), ContentType.APPLICATION_JSON); + break; + case PERFORCE: + if (config.getPerforceMode() != null) { + req.setBrowseMode("Workspace"); + } else { + req.setBrowseMode("Depot"); + } + entity = new StringEntity(convertToJson(req), StandardCharsets.UTF_8); + break; + case SHARED: + entity = new StringEntity(new Gson().toJson(req), StandardCharsets.UTF_8); + break; + case GIT: + if (req.getPrivateKey() == null || req.getPrivateKey().length < 1) { + Map content = new HashMap<>(); + content.put("url", req.getUri().getAbsoluteUrl()); + content.put("branch", config.getRemoteSrcBranch()); + entity = new StringEntity(new JSONObject(content).toString(), StandardCharsets.UTF_8); + } else { + isSSH = true; + MultipartEntityBuilder builder = MultipartEntityBuilder.create(); + builder.addTextBody("url", req.getUri().getAbsoluteUrl(), ContentType.APPLICATION_JSON); + builder.addTextBody("branch", config.getRemoteSrcBranch(), ContentType.APPLICATION_JSON); //todo add branch to req OR using without this else?? + builder.addBinaryBody("privateKey", req.getPrivateKey(), ContentType.MULTIPART_FORM_DATA, null); + entity = builder.build(); + } + break; + default: + log.error("todo"); + entity = new StringEntity("", StandardCharsets.UTF_8); + + } + if (isScanWithSettingsSupported()) { + createRemoteSourceRequest(projectId, entity, type.value(), isSSH); + ScanWithSettingsResponse response = scanWithSettings(null, projectId, true); + return response.getId(); + } else { + configureScanSettings(projectId); + createRemoteSourceRequest(projectId, entity, type.value(), isSSH); + return createScan(projectId); + } + } + + + private void configureScanSettings(long projectId) throws IOException { + ScanSettingResponse scanSettingResponse = getScanSetting(projectId); ScanSettingRequest scanSettingRequest = new ScanSettingRequest(); - scanSettingRequest.setEngineConfigurationId(scanSettingResponse.getEngineConfiguration().getId());//todo check for null + scanSettingRequest.setEngineConfigurationId(scanSettingResponse.getEngineConfiguration().getId()); scanSettingRequest.setProjectId(projectId); scanSettingRequest.setPresetId(config.getPresetId()); if (config.getEngineConfigurationId() != null) { @@ -95,143 +319,209 @@ long createSASTScan(long projectId) throws IOException, CxClientException { } //Define createSASTScan settings defineScanSetting(scanSettingRequest); - - //prepare sources for scan - if (config.getZipFile() == null) { - log.info("Zipping sources"); - File zipTempFile = CxZipUtils.zipWorkspaceFolder(config, MAX_ZIP_SIZE_BYTES, log); - //Upload zipped source file - uploadZipFile(zipTempFile, projectId); - deleteTempZipFile(zipTempFile, log); - } else { - uploadZipFile(config.getZipFile(), projectId); - } - - //Start a new createSASTScan - log.info("Uploading zip file"); - CreateScanRequest scanRequest = new CreateScanRequest(projectId, config.getIncremental(), config.getPublic(), config.getForceScan(), config.getScanComment() == null ? "" : config.getScanComment()); - log.info("Sending SAST scan request"); - CxID createScanResponse = createScan(scanRequest); - log.info(String.format("SAST Scan created successfully. Link to project state: " + config.getUrl() + LINK_FORMAT, projectId)); - - return createScanResponse.getId(); } //GET SAST results + reports - public SASTResults waitForSASTResults(long scanId, long projectId) throws InterruptedException, IOException, CxClientException { - SASTResults sastResults; - - log.info("------------------------------------Get CxSAST Results:-----------------------------------"); - //wait for SAST scan to finish - log.info("Waiting for CxSAST scan to finish."); - sastWaiter.waitForTaskToFinish(Long.toString(scanId), config.getSastScanTimeoutInMinutes() * 60, log); - log.info("Retrieving SAST scan results"); - - //retrieve SAST scan results - sastResults = retrieveSASTResults(scanId, projectId); - /* if (config.getEnablePolicyViolations()) { - resolveSASTViolation(sastResults, projectId); - }*/ - SASTUtils.printSASTResultsToConsole(sastResults, config.getEnablePolicyViolations(), log); - - //PDF report - if (config.getGeneratePDFReport()) { - log.info("Generating PDF report"); - byte[] pdfReport = getScanReport(sastResults.getScanId(), ReportType.PDF, CONTENT_TYPE_APPLICATION_PDF_V1); - sastResults.setPDFReport(pdfReport); - if (config.getReportsDir() != null) { - String pdfFileName = writePDFReport(pdfReport, config.getReportsDir(), log); - sastResults.setPdfFileName(pdfFileName); + @Override + public Results waitForScanResults() { + try { + log.info("------------------------------------Get CxSAST Results:-----------------------------------"); + //wait for SAST scan to finish + log.info("Waiting for CxSAST scan to finish."); + sastWaiter.waitForTaskToFinish(Long.toString(scanId), config.getSastScanTimeoutInMinutes() * 60, log); + log.info("Retrieving SAST scan results"); + + //retrieve SAST scan results + sastResults = retrieveSASTResults(scanId, projectId); + if (config.getEnablePolicyViolations()) { + resolveSASTViolation(sastResults, projectId); + } + SASTUtils.printSASTResultsToConsole(sastResults, config.getEnablePolicyViolations(), log); + + //PDF report + if (config.getGeneratePDFReport()) { + log.info("Generating PDF report"); + byte[] pdfReport = getScanReport(sastResults.getScanId(), ReportType.PDF, CONTENT_TYPE_APPLICATION_PDF_V1); + sastResults.setPDFReport(pdfReport); + if (config.getReportsDir() != null) { + String now = new SimpleDateFormat("dd_MM_yyyy-HH_mm_ss").format(new Date()); + String pdfFileName = PDF_REPORT_NAME + "_" + now + ".pdf"; + String pdfLink = writePDFReport(pdfReport, config.getReportsDir(), pdfFileName, log); + sastResults.setSastPDFLink(pdfLink); + sastResults.setPdfFileName(pdfFileName); + } } + // CLI report/s + else if (!config.getReports().isEmpty()) { + for (Map.Entry report : config.getReports().entrySet()) { + if (report != null) { + log.info("Generating " + report.getKey().value() + " report"); + byte[] scanReport = getScanReport(sastResults.getScanId(), report.getKey(), CONTENT_TYPE_APPLICATION_PDF_V1); + writeReport(scanReport, report.getValue(), log); + if (report.getKey().value().equals("PDF")) { + sastResults.setPDFReport(scanReport); + sastResults.setPdfFileName(report.getValue()); + } + } + } + } + } catch (Exception e) { + log.error(e.getMessage()); + sastResults.setException(new CxClientException(e)); } + return sastResults; } private void resolveSASTViolation(SASTResults sastResults, long projectId) { try { - for (Policy policy : getProjectViolations(httpClient, config.getCxARMUrl(), projectId, SAST.value())) { - sastResults.getSastPolicies().add(policy.getPolicyName()); - sastResults.addAllViolations(policy.getViolations()); - } - }catch (Exception ex) { - log.error("CxARM is not available. Policy violations for SAST cannot be calculated: " + ex.getMessage()); + cxARMWaiter.waitForTaskToFinish(Long.toString(projectId), cxARMTimeoutSec, log); + getProjectViolatedPolicies(httpClient, config.getCxARMUrl(), projectId, SAST.value()) + .forEach(sastResults::addPolicy); + } catch (Exception ex) { + throw new CxClientException("CxARM is not available. Policy violations for SAST cannot be calculated: " + ex.getMessage()); } } - private SASTResults retrieveSASTResults(long scanId, long projectId) throws CxClientException, IOException, InterruptedException { - SASTResults sastResults = new SASTResults(); + private SASTResults retrieveSASTResults(long scanId, long projectId) throws IOException { + + SASTStatisticsResponse statisticsResults = getScanStatistics(scanId); + sastResults.setResults(scanId, statisticsResults, config.getUrl(), projectId); //SAST detailed report - byte[] cxReport = getScanReport(sastResults.getScanId(), ReportType.XML, CONTENT_TYPE_APPLICATION_XML_V1); - CxXMLResults reportObj = convertToXMLResult(cxReport); - sastResults.setScanDetailedReport(reportObj); - sastResults.setRawXMLReport(cxReport); + if (config.getGenerateXmlReport() == null || config.getGenerateXmlReport()) { + byte[] cxReport = getScanReport(sastResults.getScanId(), ReportType.XML, CONTENT_TYPE_APPLICATION_XML_V1); + CxXMLResults reportObj = convertToXMLResult(cxReport); + sastResults.setScanDetailedReport(reportObj); + sastResults.setRawXMLReport(cxReport); + } sastResults.setSastResultsReady(true); return sastResults; } - SASTResults getLatestSASTResults(long projectId) throws IOException, CxClientException, InterruptedException { - log.info("---------------------------------Get Last CxSAST Results:--------------------------------"); - List scanList = getLatestSASTStatus(projectId); - for (LastScanResponse s : scanList) { - if (CurrentStatus.FINISHED.value().equals(s.getStatus().getName())) { - return retrieveSASTResults(s.getId(), projectId); + @Override + public SASTResults getLatestScanResults() { + sastResults = new SASTResults(); + try { + log.info("---------------------------------Get Last CxSAST Results:--------------------------------"); + List scanList = getLatestSASTStatus(projectId); + for (LastScanResponse s : scanList) { + if (CurrentStatus.FINISHED.value().equals(s.getStatus().getName())) { + return retrieveSASTResults(s.getId(), projectId); + } } + } catch (Exception e) { + log.error(e.getMessage()); + sastResults.setException(new CxClientException(e)); } - return new SASTResults(); + return sastResults; } //Cancel SAST Scan - public void cancelSASTScan(long scanId) throws IOException, CxClientException { + public void cancelSASTScan() throws IOException { UpdateScanStatusRequest request = new UpdateScanStatusRequest(CurrentStatus.CANCELED); String json = convertToJson(request); - StringEntity entity = new StringEntity(json); - httpClient.patchRequest(SAST_QUEUE_SCAN_STATUS.replace("{scanId}", Long.toString(scanId)), CONTENT_TYPE_APPLICATION_JSON_V1, entity, 200, "cancel SAST scan"); + StringEntity entity = new StringEntity(json, StandardCharsets.UTF_8); + httpClient.patchRequest(SAST_QUEUE_SCAN_STATUS.replace(SCAN_ID_PATH_PARAM, Long.toString(scanId)), CONTENT_TYPE_APPLICATION_JSON_V1, entity, 200, "cancel SAST scan"); log.info("SAST Scan canceled. (scanId: " + scanId + ")"); } - //**------ Private Methods ------**// + private boolean projectHasQueuedScans(long projectId) throws IOException { + List queuedScans = getQueueScans(projectId); + for (ResponseQueueScanStatus scan : queuedScans) { + if (isStatusToAvoid(scan.getStage().getValue()) && scan.getProject().getId() == projectId) { + return true; + } + } + return false; + } - private ScanSettingResponse getScanSetting(long projectId) throws IOException, CxClientException { - return httpClient.getRequest(SAST_GET_SCAN_SETTINGS.replace("{projectId}", Long.toString(projectId)), CONTENT_TYPE_APPLICATION_JSON_V1, ScanSettingResponse.class, 200, "Scan setting", false); + private boolean isStatusToAvoid(String status) { + QueueStatus qStatus = QueueStatus.valueOf(status); + + switch (qStatus) { + case New: + case PreScan: + case SourcePullingAndDeployment: + case Queued: + case Scanning: + case PostScan: + return true; + default: + return false; + } + } + + public ScanSettingResponse getScanSetting(long projectId) throws IOException { + return httpClient.getRequest(SAST_GET_SCAN_SETTINGS.replace(PROJECT_ID_PATH_PARAM, Long.toString(projectId)), CONTENT_TYPE_APPLICATION_JSON_V1, ScanSettingResponse.class, 200, "Scan setting", false); } - private void defineScanSetting(ScanSettingRequest scanSetting) throws IOException, CxClientException { - StringEntity entity = new StringEntity(convertToJson(scanSetting)); + private void defineScanSetting(ScanSettingRequest scanSetting) throws IOException { + StringEntity entity = new StringEntity(convertToJson(scanSetting), StandardCharsets.UTF_8); httpClient.putRequest(SAST_UPDATE_SCAN_SETTINGS, CONTENT_TYPE_APPLICATION_JSON_V1, entity, CxID.class, 200, "define scan setting"); } - private void uploadZipFile(File zipFile, long projectId) throws CxClientException, IOException { - InputStreamBody streamBody = new InputStreamBody(new FileInputStream(zipFile.getAbsoluteFile()), ContentType.APPLICATION_OCTET_STREAM, "zippedSource"); - MultipartEntityBuilder builder = MultipartEntityBuilder.create(); - builder.setMode(HttpMultipartMode.BROWSER_COMPATIBLE); - builder.addPart("zippedSource", streamBody); - HttpEntity entity = builder.build(); - httpClient.postRequest(SAST_ZIP_ATTACHMENTS.replace("{projectId}", Long.toString(projectId)), null, entity, null, 204, "upload ZIP file"); + private void excludeProjectSettings(long projectId) throws IOException { + String excludeFoldersPattern = Arrays.stream(config.getSastFolderExclusions().split(",")).map(String::trim).collect(Collectors.joining(",")); + String excludeFilesPattern = Arrays.stream(config.getSastFilterPattern().split(",")).map(String::trim).map(file -> file.replace("!**/", "")).collect(Collectors.joining(",")); + ExcludeSettingsRequest excludeSettingsRequest = new ExcludeSettingsRequest(excludeFoldersPattern, excludeFilesPattern); + StringEntity entity = new StringEntity(convertToJson(excludeSettingsRequest), StandardCharsets.UTF_8); + log.info("Exclude folders pattern: " + excludeFoldersPattern); + log.info("Exclude files pattern: " + excludeFilesPattern); + httpClient.putRequest(String.format(SAST_EXCLUDE_FOLDERS_FILES_PATTERNS, projectId), CONTENT_TYPE_APPLICATION_JSON_V1, entity, null, 200, "exclude project's settings"); + } + + private void uploadZipFile(byte[] zipFile, long projectId) throws CxClientException, IOException { + log.info("Uploading zip file"); + + try (InputStream is = new ByteArrayInputStream(zipFile)) { + InputStreamBody streamBody = new InputStreamBody(is, ContentType.APPLICATION_OCTET_STREAM, ZIPPED_SOURCE); + MultipartEntityBuilder builder = MultipartEntityBuilder.create(); + builder.setMode(HttpMultipartMode.BROWSER_COMPATIBLE); + builder.addPart(ZIPPED_SOURCE, streamBody); + HttpEntity entity = builder.build(); + httpClient.postRequest(SAST_ZIP_ATTACHMENTS.replace(PROJECT_ID_PATH_PARAM, Long.toString(projectId)), null, new BufferedHttpEntity(entity), null, 204, "upload ZIP file"); + } + } + + private long createScan(long projectId) throws IOException { + CreateScanRequest scanRequest = new CreateScanRequest(projectId, config.getIncremental(), config.getPublic(), config.getForceScan(), config.getScanComment() == null ? "" : config.getScanComment()); + + log.info("Sending SAST scan request"); + StringEntity entity = new StringEntity(convertToJson(scanRequest), StandardCharsets.UTF_8); + CxID createScanResponse = httpClient.postRequest(SAST_CREATE_SCAN, CONTENT_TYPE_APPLICATION_JSON_V1, entity, CxID.class, 201, "create new SAST Scan"); + log.info(String.format("SAST Scan created successfully. Link to project state: " + config.getUrl() + LINK_FORMAT, projectId)); + + return createScanResponse.getId(); + } + + private CxID createRemoteSourceRequest(long projectId, HttpEntity entity, String sourceType, boolean isSSH) throws IOException { + return httpClient.postRequest(String.format(SAST_CREATE_REMOTE_SOURCE_SCAN, projectId, sourceType, isSSH ? "ssh" : ""), isSSH ? null : CONTENT_TYPE_APPLICATION_JSON_V1, + entity, CxID.class, 204, "create " + sourceType + " remote source scan setting"); + } - private CxID createScan(CreateScanRequest request) throws CxClientException, IOException { - StringEntity entity = new StringEntity(convertToJson(request)); - return httpClient.postRequest(SAST_CREATE_SCAN, CONTENT_TYPE_APPLICATION_JSON_V1, entity, CxID.class, 201, "create new SAST Scan"); + private SASTStatisticsResponse getScanStatistics(long scanId) throws IOException { + return httpClient.getRequest(SAST_SCAN_RESULTS_STATISTICS.replace(SCAN_ID_PATH_PARAM, Long.toString(scanId)), CONTENT_TYPE_APPLICATION_JSON_V1, SASTStatisticsResponse.class, 200, "SAST scan statistics", false); } - private SASTStatisticsResponse getScanStatistics(long scanId) throws CxClientException, IOException { - return httpClient.getRequest(SAST_SCAN_RESULTS_STATISTICS.replace("{scanId}", Long.toString(scanId)), CONTENT_TYPE_APPLICATION_JSON_V1, SASTStatisticsResponse.class, 200, "SAST scan statistics", false); + public List getLatestSASTStatus(long projectId) throws IOException { + return (List) httpClient.getRequest(SAST_GET_PROJECT_SCANS.replace(PROJECT_ID_PATH_PARAM, Long.toString(projectId)), CONTENT_TYPE_APPLICATION_JSON_V1, LastScanResponse.class, 200, "last SAST scan ID", true); } - private List getLatestSASTStatus(long projectId) throws CxClientException, IOException { - return (List) httpClient.getRequest(SAST_GET_PROJECT_SCANS.replace("{projectId}", Long.toString(projectId)), CONTENT_TYPE_APPLICATION_JSON_V1, LastScanResponse.class, 200, "last SAST scan ID", true); + private List getQueueScans(long projectId) throws IOException { + return (List) httpClient.getRequest(SAST_GET_QUEUED_SCANS.replace(PROJECT_ID_PATH_PARAM, Long.toString(projectId)), CONTENT_TYPE_APPLICATION_JSON_V1, ResponseQueueScanStatus.class, 200, "scans in the queue. (projectId: )" + projectId, true); } - private CreateReportResponse createScanReport(CreateReportRequest reportRequest) throws CxClientException, IOException { - StringEntity entity = new StringEntity(convertToJson(reportRequest)); + private CreateReportResponse createScanReport(CreateReportRequest reportRequest) throws IOException { + StringEntity entity = new StringEntity(convertToJson(reportRequest), StandardCharsets.UTF_8); return httpClient.postRequest(SAST_CREATE_REPORT, CONTENT_TYPE_APPLICATION_JSON_V1, entity, CreateReportResponse.class, 202, "to create " + reportRequest.getReportType() + " scan report"); } - private byte[] getScanReport(long scanId, ReportType reportType, String contentType) throws CxClientException, IOException, InterruptedException { + private byte[] getScanReport(long scanId, ReportType reportType, String contentType) throws IOException { CreateReportRequest reportRequest = new CreateReportRequest(scanId, reportType.name()); CreateReportResponse createReportResponse = createScanReport(reportRequest); int reportId = createReportResponse.getReportId(); @@ -240,13 +530,13 @@ private byte[] getScanReport(long scanId, ReportType reportType, String contentT return getReport(reportId, contentType); } - private byte[] getReport(long reportId, String contentType) throws CxClientException, IOException { + private byte[] getReport(long reportId, String contentType) throws IOException { return httpClient.getRequest(SAST_GET_REPORT.replace("{reportId}", Long.toString(reportId)), contentType, byte[].class, 200, " scan report: " + reportId, false); } //SCAN Waiter - overload methods - private ResponseQueueScanStatus getSASTScanStatus(String scanId) throws CxClientException, IOException { - ResponseQueueScanStatus scanStatus = httpClient.getRequest(SAST_QUEUE_SCAN_STATUS.replace("{scanId}", scanId), CONTENT_TYPE_APPLICATION_JSON_V1, ResponseQueueScanStatus.class, 200, "SAST scan status", false); + public ResponseQueueScanStatus getSASTScanStatus(String scanId) throws IOException { + ResponseQueueScanStatus scanStatus = httpClient.getRequest(SAST_QUEUE_SCAN_STATUS.replace(SCAN_ID_PATH_PARAM, scanId), CONTENT_TYPE_APPLICATION_JSON_V1, ResponseQueueScanStatus.class, 200, "SAST scan status", false); String currentStatus = scanStatus.getStage().getValue(); if (CurrentStatus.FAILED.value().equals(currentStatus) || CurrentStatus.CANCELED.value().equals(currentStatus) || @@ -262,54 +552,61 @@ private ResponseQueueScanStatus getSASTScanStatus(String scanId) throws CxClient } private void printSASTProgress(ResponseQueueScanStatus scanStatus, long startTime) { - long elapsedSec = System.currentTimeMillis() / 1000 - startTime; - long hours = elapsedSec / 3600; - long minutes = elapsedSec % 3600 / 60; - long seconds = elapsedSec % 60; - String hoursStr = (hours < 10) ? ("0" + Long.toString(hours)) : (Long.toString(hours)); - String minutesStr = (minutes < 10) ? ("0" + Long.toString(minutes)) : (Long.toString(minutes)); - String secondsStr = (seconds < 10) ? ("0" + Long.toString(seconds)) : (Long.toString(seconds)); + String timestamp = ShragaUtils.getTimestampSince(startTime); String prefix = (scanStatus.getTotalPercent() < 10) ? " " : ""; - log.info("Waiting for SAST scan results. Elapsed time: " + hoursStr + ":" + minutesStr + ":" + secondsStr + ". " + prefix + + log.info("Waiting for SAST scan results. Elapsed time: " + timestamp + ". " + prefix + scanStatus.getTotalPercent() + "% processed. Status: " + scanStatus.getStage().getValue() + "."); } - private ResponseQueueScanStatus resolveSASTStatus(ResponseQueueScanStatus scanStatus) throws CxClientException { - if (Status.SUCCEEDED == scanStatus.getBaseStatus()) { - log.info("SAST scan finished successfully."); - return scanStatus; + private ResponseQueueScanStatus resolveSASTStatus(ResponseQueueScanStatus scanStatus) { + if (scanStatus != null) { + if (Status.SUCCEEDED == scanStatus.getBaseStatus()) { + log.info("SAST scan finished successfully."); + return scanStatus; + } else { + throw new CxClientException("SAST scan cannot be completed. status [" + scanStatus.getStage().getValue() + "]: " + scanStatus.getStageDetails()); + } } else { - throw new CxClientException("SAST scan cannot be completed. status [" + scanStatus.getStage().getValue() + "]: " + scanStatus.getStageDetails()); + throw new CxClientException("SAST scan cannot be completed."); } } - //Report Waiter - overload methods - private ReportStatus getReportStatus(String reportId) throws CxClientException, IOException { - ReportStatus reportStatus = httpClient.getRequest(SAST_GET_REPORT_STATUS.replace("{reportId}", reportId), CONTENT_TYPE_APPLICATION_JSON_V1, ReportStatus.class, 200, " report status", false); - - String currentStatus = reportStatus.getStatus().getValue(); - if (currentStatus.equals(ReportStatusEnum.INPROCESS.value())) { - reportStatus.setBaseStatus(Status.IN_PROGRESS); - } else if (currentStatus.equals(ReportStatusEnum.FAILED.value())) { - reportStatus.setBaseStatus(Status.FAILED); - } else { - reportStatus.setBaseStatus(Status.SUCCEEDED); - } - - return reportStatus; + @Override + public Results initiateScan() { + sastResults = new SASTResults(); + createSASTScan(projectId); + return sastResults; } - private void printReportProgress(ReportStatus reportStatus, long startTime) { - String reportType = reportStatus.getContentType().replace("application/", ""); - log.info("Waiting for server to generate " + reportType + " report. " + (startTime + reportTimeoutSec - (System.currentTimeMillis() / 1000)) + " seconds left to timeout"); + private boolean isScanWithSettingsSupported() { + try { + HashMap swaggerResponse = this.httpClient.getRequest(SWAGGER_LOCATION, CONTENT_TYPE_APPLICATION_JSON, HashMap.class, 200, "SAST scan status", false); + return swaggerResponse.toString().contains("/sast/scanWithSettings"); + } catch (Exception e) { + return false; + } } - private ReportStatus resolveReportStatus(ReportStatus reportStatus) throws CxClientException { - if (Status.SUCCEEDED == reportStatus.getBaseStatus()) { - return reportStatus; - } else { - throw new CxClientException("Generation of scan report [id=" + reportStatus.getBaseId() + "] failed."); + private ScanWithSettingsResponse scanWithSettings(byte[] zipFile, long projectId, boolean isRemote) throws IOException { + log.info("Uploading zip file"); + MultipartEntityBuilder builder = MultipartEntityBuilder.create(); + builder.setMode(HttpMultipartMode.BROWSER_COMPATIBLE); + if(!isRemote) { + try (InputStream is = new ByteArrayInputStream(zipFile)) { + InputStreamBody streamBody = new InputStreamBody(is, ContentType.APPLICATION_OCTET_STREAM, ZIPPED_SOURCE); + builder.addPart(ZIPPED_SOURCE, streamBody); + } } + builder.addTextBody("projectId",Long.toString(projectId), ContentType.APPLICATION_JSON); + builder.addTextBody("overrideProjectSetting", super.isIsNewProject()?"true":"false" , ContentType.APPLICATION_JSON); + builder.addTextBody("isIncremental", config.getIncremental().toString() , ContentType.APPLICATION_JSON); + builder.addTextBody("isPublic", config.getPublic().toString() , ContentType.APPLICATION_JSON); + builder.addTextBody("forceScan", config.getForceScan().toString() , ContentType.APPLICATION_JSON); + builder.addTextBody("presetId", config.getPresetId().toString() , ContentType.APPLICATION_JSON); + builder.addTextBody("comment", config.getScanComment()==null?"":config.getScanComment() , ContentType.APPLICATION_JSON); + builder.addTextBody("engineConfigurationId", config.getEngineConfigurationId()!=null?config.getEngineConfigurationId().toString():ENGINE_CONFIGURATION_ID_DEFAULT , ContentType.APPLICATION_JSON); + HttpEntity entity = builder.build(); + return httpClient.postRequest(SCAN_WITH_SETTINGS_URL, null, new BufferedHttpEntity(entity), ScanWithSettingsResponse.class, 201, "upload ZIP file"); } } diff --git a/src/main/java/com/cx/restclient/CxShragaClient.java b/src/main/java/com/cx/restclient/CxShragaClient.java deleted file mode 100644 index 29eecff5..00000000 --- a/src/main/java/com/cx/restclient/CxShragaClient.java +++ /dev/null @@ -1,299 +0,0 @@ -package com.cx.restclient; - -import com.cx.restclient.common.summary.SummaryUtils; -import com.cx.restclient.configuration.CxScanConfig; -import com.cx.restclient.cxArm.dto.CxArmConfig; -import com.cx.restclient.dto.Team; -import com.cx.restclient.dto.ThresholdResult; -import com.cx.restclient.exception.CxClientException; -import com.cx.restclient.exception.CxHTTPClientException; -import com.cx.restclient.httpClient.CxHttpClient; -import com.cx.restclient.osa.dto.OSAResults; -import com.cx.restclient.sast.dto.*; -import org.apache.http.client.HttpResponseException; -import org.apache.http.entity.StringEntity; -import org.slf4j.Logger; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URLEncoder; -import java.util.List; -import java.util.Properties; - -import static com.cx.restclient.common.CxPARAM.*; -import static com.cx.restclient.common.ShragaUtils.isThresholdExceeded; -import static com.cx.restclient.httpClient.utils.ContentType.CONTENT_TYPE_APPLICATION_JSON_V1; -import static com.cx.restclient.httpClient.utils.HttpClientHelper.convertToJson; -import static com.cx.restclient.sast.utils.SASTParam.*; - -/** - * Created by Galn on 05/02/2018. - */ -//SHRAGA -//System Holistic Rest Api Generic Application -public class CxShragaClient { - - private CxHttpClient httpClient; - private Logger log; - private CxScanConfig config; - private long projectId; - - private CxSASTClient sastClient; - private CxOSAClient osaClient; - private long sastScanId; - private String osaScanId; - private SASTResults sastResults = new SASTResults(); - private OSAResults osaResults = new OSAResults(); - - - public CxShragaClient(CxScanConfig config, Logger log) throws MalformedURLException { - this.config = config; - this.log = log; - this.httpClient = new CxHttpClient( - config.getUrl(), - config.getUsername(), - config.getPassword(), - config.getCxOrigin(), - config.isDisableCertificateValidation(), log); - sastClient = new CxSASTClient(httpClient, log, config); - osaClient = new CxOSAClient(httpClient, log, config); - } - - public CxShragaClient(String serverUrl, String username, String password, String origin, boolean disableCertificateValidation, Logger log) throws MalformedURLException { - this(new CxScanConfig(serverUrl, username, password, origin, disableCertificateValidation), log); - } - - //API Scans methods - public void init() throws CxClientException, IOException { - log.info("Initializing Cx client"); - login(); - if (config.getSastEnabled()) { - resolvePreset(); - } - if (config.getEnablePolicyViolations()){ - resolveCxARMUrl(); - } - resolveTeam(); - resolveProject(); - } - - public long createSASTScan() throws IOException, CxClientException { - sastScanId = sastClient.createSASTScan(projectId); - sastResults.setSastScanLink(config.getUrl(), sastScanId, projectId); - return sastScanId; - } - - public String createOSAScan() throws IOException, CxClientException { - osaScanId = osaClient.createOSAScan(projectId); - osaResults.setOsaProjectSummaryLink(config.getUrl(), projectId); - return osaScanId; - } - - public void cancelSASTScan() throws IOException, CxClientException { - sastClient.cancelSASTScan(sastScanId); - } - - public SASTResults waitForSASTResults() throws InterruptedException, CxClientException, IOException { - sastResults = sastClient.waitForSASTResults(sastScanId, projectId); - return sastResults; - } - - public SASTResults getLatestSASTResults() throws InterruptedException, CxClientException, IOException { - sastResults = sastClient.getLatestSASTResults(projectId); - return sastResults; - } - - public OSAResults waitForOSAResults() throws InterruptedException, CxClientException, IOException { - osaResults = osaClient.getOSAResults(osaScanId, projectId); - return osaResults; - } - - public OSAResults getLatestOSAResults() throws InterruptedException, CxClientException, IOException { - osaResults = osaClient.getLatestOSAResults(projectId); - return osaResults; - } - - public ThresholdResult getThresholdResult() { - StringBuilder res = new StringBuilder(""); - boolean isFail = isThresholdExceeded(config, sastResults, osaResults, res); - return new ThresholdResult(isFail, res.toString()); - } - - public boolean isPolicyViolated(StringBuilder failDescription) { - boolean isPolicyViolated = config.getEnablePolicyViolations() && osaResults.getOsaViolations().size() > 0; - if(isPolicyViolated) { - failDescription.append("Project policy status: violated").append("\n");; - } - return isPolicyViolated; - } - - - private CxArmConfig getCxARMConfig() throws IOException, CxClientException { - return httpClient.getRequest(CX_ARM_URL, CONTENT_TYPE_APPLICATION_JSON_V1, CxArmConfig.class, 200, "CxARM URL", false); - } - - public String generateHTMLSummary() throws Exception { - return SummaryUtils.generateSummary(sastResults, osaResults, config); - } - - public String generateHTMLSummary(SASTResults sastResults, OSAResults osaResults) throws Exception { - return SummaryUtils.generateSummary(sastResults, osaResults, config); - } - - public List getAllProjects() throws IOException, CxClientException { - List projects = null; - try { - projects = (List) httpClient.getRequest(SAST_GET_All_PROJECTS, CONTENT_TYPE_APPLICATION_JSON_V1, Project.class, 200, "all projects", true); - } catch (HttpResponseException ex) { - if (ex.getStatusCode() != 404) { - throw ex; - } - } - return projects; - } - - public void close() { - httpClient.close(); - } - - //HELP config Methods - public void login() throws IOException, CxClientException { - // perform login to server - log.info("Logging into the Checkmarx service."); - httpClient.login(); - } - - public String getTeamIdByName(String teamName) throws CxClientException, IOException { - List allTeams = getTeamList(); - for (Team team : allTeams) { - if (team.getFullName().equalsIgnoreCase(teamName)) { //TODO caseSenesitive - return team.getId(); - } - } - throw new CxClientException("Could not resolve team ID from team name: " + teamName); - } - - public String getTeamNameById(String teamId) throws CxClientException, IOException { - List allTeams = getTeamList(); - for (Team team : allTeams) { - if (teamId.equals(team.getId())) { - return team.getFullName(); - } - } - throw new CxClientException("Could not resolve team name from id: " + teamId); - } - - public int getPresetIdByName(String presetName) throws CxClientException, IOException { - List allPresets = getPresetList(); - for (Preset preset : allPresets) { - if (preset.getName().equalsIgnoreCase(presetName)) { //TODO caseSenesitive- checkkk - return preset.getId(); - } - } - - throw new CxClientException("Could not resolve preset ID from preset name: " + presetName); - } - - public List getTeamList() throws IOException, CxClientException { - return (List) httpClient.getRequest(CXTEAMS, CONTENT_TYPE_APPLICATION_JSON_V1, Team.class, 200, "team list", true); - } - - public Preset getPresetById(int presetId) throws IOException, CxClientException { - return httpClient.getRequest(CXPRESETS + "/" + presetId, CONTENT_TYPE_APPLICATION_JSON_V1, Preset.class, 200, "preset by id", false); - } - - public List getPresetList() throws IOException, CxClientException { - return (List) httpClient.getRequest(CXPRESETS, CONTENT_TYPE_APPLICATION_JSON_V1, Preset.class, 200, "preset list", true); - } - - public List getConfigurationSetList() throws IOException, CxClientException { - return (List) httpClient.getRequest(SAST_ENGINE_CONFIG, CONTENT_TYPE_APPLICATION_JSON_V1, CxNameObj.class, 200, "engine configurations", true); - } - - public void setOsaFSAProperties(Properties fsaConfig) { //For CxMaven plugin - config.setOsaFsaConfig(fsaConfig); - } - - //Private methods - private void resolveTeam() throws CxClientException, IOException { - if (config.getTeamId() == null) { - config.setTeamId(getTeamIdByName(config.getTeamPath())); - } - printTeamPath(); - } - - private void resolveCxARMUrl() { - try { - this.config.setCxARMUrl(getCxARMConfig().getCxARMPolicyURL()); - } catch (Exception ex) { - log.error("CxARM is not available. Policy violations cannot be calculated: " + ex.getMessage()); - } - } - - private void resolvePreset() throws CxClientException, IOException { - if (config.getPresetId() == null) { - config.setPresetId(getPresetIdByName(config.getPresetName())); - } - printPresetName(); - } - - private void printPresetName() { - try { - String presetName = config.getPresetName(); - if (presetName == null) { - presetName = getPresetById(config.getPresetId()).getName(); - } - log.info("preset name: " + presetName); - } catch (Exception e) { - } - } - - private void printTeamPath() { - try { - String teamPath = config.getTeamPath(); - if (teamPath == null) { - teamPath = getTeamNameById(config.getTeamId()); - } - log.info("full team path: " + teamPath); - } catch (Exception e) { - } - } - - private void resolveProject() throws IOException, CxClientException { - List projects = getProjectByName(config.getProjectName(), config.getTeamId()); - if (projects == null || projects.isEmpty()) { // Project is new - if (config.getDenyProject()) { - String errMsg = "Creation of the new project [" + config.getProjectName() + "] is not authorized. " + - "Please use an existing project. \nYou can enable the creation of new projects by disabling" + "" + - " the Deny new Checkmarx projects creation checkbox in the Checkmarx plugin global settings.\n"; - throw new CxClientException(errMsg); - } - //Create newProject - CreateProjectRequest request = new CreateProjectRequest(config.getProjectName(), config.getTeamId(), config.getPublic()); - projectId = createNewProject(request).getId(); - - } else { - projectId = projects.get(0).getId(); - } - } - - private List getProjectByName(String projectName, String teamId) throws IOException, CxClientException { - projectName = URLEncoder.encode(projectName, "UTF-8"); - String projectNamePath = SAST_GET_PROJECT.replace("{name}", projectName).replace("{teamId}", teamId); - List projects = null; - try { - projects = (List) httpClient.getRequest(projectNamePath, CONTENT_TYPE_APPLICATION_JSON_V1, Project.class, 200, "project by name: " + projectName, true); - } catch (CxHTTPClientException ex) { - if (ex.getStatusCode() != 404) { - throw ex; - } - } - return projects; - } - - private Project createNewProject(CreateProjectRequest request) throws CxClientException, IOException { - String json = convertToJson(request); - StringEntity entity = new StringEntity(json); - return httpClient.postRequest(CREATE_PROJECT, CONTENT_TYPE_APPLICATION_JSON_V1, entity, Project.class, 201, "create new project: " + request.getName()); - } -} \ No newline at end of file diff --git a/src/main/java/com/cx/restclient/ast/AstClient.java b/src/main/java/com/cx/restclient/ast/AstClient.java new file mode 100644 index 00000000..12216565 --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/AstClient.java @@ -0,0 +1,310 @@ +package com.cx.restclient.ast; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.entity.StringEntity; +import org.slf4j.Logger; + +import com.cx.restclient.ast.dto.common.ASTConfig; +import com.cx.restclient.ast.dto.common.GitCredentials; +import com.cx.restclient.ast.dto.common.HandlerRef; +import com.cx.restclient.ast.dto.common.ProjectToScan; +import com.cx.restclient.ast.dto.common.RemoteRepositoryInfo; +import com.cx.restclient.ast.dto.common.ScanConfig; +import com.cx.restclient.ast.dto.common.ScanStartHandler; +import com.cx.restclient.ast.dto.common.StartScanRequest; +import com.cx.restclient.ast.dto.sca.AstScaConfig; +import com.cx.restclient.common.UrlUtils; +import com.cx.restclient.configuration.CxScanConfig; +import com.cx.restclient.configuration.PropertyFileLoader; +import com.cx.restclient.dto.Results; +import com.cx.restclient.dto.SourceLocationType; +import com.cx.restclient.exception.CxClientException; +import com.cx.restclient.httpClient.CxHttpClient; +import com.cx.restclient.httpClient.utils.ContentType; +import com.cx.restclient.httpClient.utils.HttpClientHelper; +import com.cx.restclient.sast.utils.State; +import com.cx.restclient.sca.dto.CxSCAScanAPIConfig; +import com.cx.restclient.sca.dto.CxSCAScanApiConfigEntry; +import com.cx.restclient.sca.dto.GetUploadUrlRequest; +import com.cx.restclient.sca.dto.ScanAPIConfigEntry; +import com.fasterxml.jackson.databind.JsonNode; + +public abstract class AstClient { + + private static final String LOCATION_HEADER = "Location"; + private static final String CREDENTIAL_TYPE_PASSWORD = "password"; + protected static final String ENCODING = StandardCharsets.UTF_8.name(); + + protected final CxScanConfig config; + protected final Logger log; + + protected CxHttpClient httpClient; + + private State state = State.SUCCESS; + + protected static final PropertyFileLoader properties = PropertyFileLoader.getDefaultInstance(); + public static final String GET_SCAN = properties.get("ast.getScan"); + public static final String CREATE_SCAN = properties.get("ast.createScan"); + public static final String GET_UPLOAD_URL = properties.get("ast.getUploadUrl"); + + public AstClient(CxScanConfig config, Logger log) { + validate(config, log); + this.config = config; + this.log = log; + } + + protected abstract String getScannerDisplayName(); + + protected abstract ScanConfig getScanConfig(); + + protected abstract HandlerRef getBranchToScan(RemoteRepositoryInfo repoInfo); + + protected abstract HttpResponse submitAllSourcesFromLocalDir(String projectId, String zipFilePath) throws IOException; + + protected abstract String getWebReportPath() throws UnsupportedEncodingException; + + protected CxHttpClient createHttpClient(String baseUrl) { + log.debug("Creating HTTP client."); + CxHttpClient client = new CxHttpClient(baseUrl, + config.getCxOrigin(), + config.getCxOriginUrl(), + config.isDisableCertificateValidation(), + false, // AST clients don't support SSO. + null, + config.isProxy(), + config.getProxyConfig(), + log, + config.getNTLM()); + //initializing Team Path to prevent null pointer in login when called from automation + client.setTeamPathHeader(""); + + return client; + } + + private void validate(CxScanConfig config, Logger log) { + if (config == null && log == null) { + throw new CxClientException("Both scan config and log must be provided."); + } + } + + protected HttpResponse sendStartScanRequest(RemoteRepositoryInfo repoInfo, + SourceLocationType sourceLocation, + String projectId) throws IOException { + log.debug("Constructing the 'start scan' request"); + + ScanStartHandler handler = getScanStartHandler(repoInfo); + + ProjectToScan project = ProjectToScan.builder() + .id(projectId) + .type(sourceLocation.getApiValue()) + .handler(handler) + .build(); + + List apiScanConfig = Collections.singletonList(getScanConfig()); + + StartScanRequest request = StartScanRequest.builder() + .project(project) + .config(apiScanConfig) + .build(); + + StringEntity entity = HttpClientHelper.convertToStringEntity(request); + + log.info("Sending the 'start scan' request."); + return httpClient.postRequest(CREATE_SCAN, ContentType.CONTENT_TYPE_APPLICATION_JSON, entity, + HttpResponse.class, HttpStatus.SC_CREATED, "start the scan"); + } + + protected HttpResponse submitSourcesFromRemoteRepo(ASTConfig config, String projectId) throws IOException { + log.info("Using remote repository flow."); + RemoteRepositoryInfo repoInfo = config.getRemoteRepositoryInfo(); + validateRepoInfo(repoInfo); + + URL sanitizedUrl = sanitize(repoInfo.getUrl()); + log.info("Repository URL: {}", sanitizedUrl); + return sendStartScanRequest(repoInfo, SourceLocationType.REMOTE_REPOSITORY, projectId); + } + + protected void waitForScanToFinish(String scanId) { + + log.info("------------------------------------Get {} Results:-----------------------------------", getScannerDisplayName()); + log.info("Waiting for {} scan to finish", getScannerDisplayName()); + + AstWaiter waiter = new AstWaiter(httpClient, config, getScannerDisplayName(), log); + waiter.waitForScanToFinish(scanId); + log.info("{} scan finished successfully. Retrieving {} scan results.", getScannerDisplayName(), getScannerDisplayName()); + } + + /** + * @param repoInfo may represent an actual git repo or a presigned URL of an uploaded archive. + */ + private ScanStartHandler getScanStartHandler(RemoteRepositoryInfo repoInfo) { + log.debug("Creating the handler object."); + + HandlerRef ref = getBranchToScan(repoInfo); + + // AST-SAST doesn't allow nulls here. + String password = StringUtils.defaultString(repoInfo.getPassword()); + String username = StringUtils.defaultString(repoInfo.getUsername()); + + GitCredentials credentials = GitCredentials.builder() + .type(CREDENTIAL_TYPE_PASSWORD) + .value(password) + .build(); + + URL effectiveRepoUrl = getEffectiveRepoUrl(repoInfo); + + // The ref/username/credentials properties are mandatory even if not specified in repoInfo. + return ScanStartHandler.builder() + .ref(ref) + .username(username) + .credentials(credentials) + .url(effectiveRepoUrl.toString()) + .build(); + } + + protected URL getEffectiveRepoUrl(RemoteRepositoryInfo repoInfo) { + return repoInfo.getUrl(); + } + + protected String getWebReportLink(String baseUrl) { + String result = null; + String warning = null; + try { + if (StringUtils.isNotEmpty(baseUrl)) { + String path = getWebReportPath(); + result = UrlUtils.parseURLToString(baseUrl, path); + } else { + warning = "Web app URL is not specified."; + } + } catch (MalformedURLException e) { + warning = "Invalid web app URL."; + } catch (Exception e) { + warning = "General error."; + } + + Optional.ofNullable(warning) + .ifPresent(warn -> log.warn("Unable to generate web report link. {}", warn)); + + return result; + } + + /** + * Removes the userinfo part of the input URL (if present), so that the URL may be logged safely. + * The URL may contain userinfo when a private repo is scanned. + */ + private static URL sanitize(URL url) throws MalformedURLException { + return new URL(url.getProtocol(), url.getHost(), url.getFile()); + } + + private void validateRepoInfo(RemoteRepositoryInfo repoInfo) { + log.debug("Validating remote repository info."); + if (repoInfo == null) { + String message = String.format( + "%s must be provided in %s configuration when using source location of type %s.", + RemoteRepositoryInfo.class.getName(), + getScannerDisplayName(), + SourceLocationType.REMOTE_REPOSITORY.name()); + + throw new CxClientException(message); + } + } + + protected String extractScanIdFrom(HttpResponse response) { + String result = null; + + log.debug("Extracting scan ID from the '{}' response header.", LOCATION_HEADER); + if (response != null && response.getLastHeader(LOCATION_HEADER) != null) { + // Expecting values like + // /api/scans/1ecffa00-0e42-49b2-8755-388b9f6a9293 + // /07e5b4b0-184a-458e-9d82-7f3da407f940 + String urlPathWithScanId = response.getLastHeader(LOCATION_HEADER).getValue(); + result = FilenameUtils.getName(urlPathWithScanId); + } + + if (StringUtils.isNotEmpty(result)) { + + log.info("Scan started successfully. Scan ID: {}", result); + } else { + throw new CxClientException("Unable to get scan ID."); + } + return result; + } + + protected void handleInitError(Exception e, Results results) { + String message = String.format("Failed to init %s client. %s", getScannerDisplayName(), e.getMessage()); + log.error(message); + setState(State.FAILED); + results.setException(new CxClientException(message, e)); + } + + protected HttpResponse initiateScanForUpload(String projectId, byte[] zipFile, ASTConfig scanConfig) throws IOException { + String uploadedArchiveUrl = getSourcesUploadUrl(scanConfig); + String cleanPath = uploadedArchiveUrl.split("\\?")[0]; + log.info("Uploading to: {}", cleanPath); + uploadArchive(zipFile, uploadedArchiveUrl); + + //delete only if path not specified in the config + //If zipFilePath is specified in config, it means that the user has prepared the zip file themselves. The user obviously doesn't want this file to be deleted. + //If zipFilePath is NOT specified, Common Client will create the zip itself. After uploading the zip, Common Client should clean after itself (delete the zip file that it created). + + RemoteRepositoryInfo uploadedFileInfo = new RemoteRepositoryInfo(); + uploadedFileInfo.setUrl(new URL(uploadedArchiveUrl)); + + return sendStartScanRequest(uploadedFileInfo, SourceLocationType.LOCAL_DIRECTORY, projectId); + } + + private String getSourcesUploadUrl(ASTConfig scanConfig) throws IOException { + JsonNode response; + if (scanConfig instanceof AstScaConfig) { + AstScaConfig scaConfig = (AstScaConfig) scanConfig; + boolean includeSources = scaConfig.isIncludeSources(); + CxSCAScanAPIConfig scaApiConfig = CxSCAScanAPIConfig.builder() + .includeSourceCode(includeSources ? "true" : "false").build(); + CxSCAScanApiConfigEntry configentry = CxSCAScanApiConfigEntry.builder().type("sca").value(scaApiConfig) + .build(); + List scanconfigEntry = Collections.singletonList(configentry); + + GetUploadUrlRequest request = GetUploadUrlRequest.builder(). + config(scanconfigEntry). + build(); + + StringEntity entity = HttpClientHelper.convertToStringEntity(request); + response = httpClient.postRequest(GET_UPLOAD_URL, null, entity, JsonNode.class, HttpStatus.SC_OK, + "get upload URL for sources"); + } + else { + response = httpClient.postRequest(GET_UPLOAD_URL, null, null, JsonNode.class, + HttpStatus.SC_OK, "get upload URL for sources"); + } + + if (response == null || response.get("url") == null) { + throw new CxClientException("Unable to get the upload URL."); + } + + return response.get("url").asText(); + } + + protected abstract void uploadArchive(byte[] source, String uploadUrl) throws IOException; + + + + public State getState() { + return state; + } + + public void setState(State state) { + this.state = state; + } +} diff --git a/src/main/java/com/cx/restclient/ast/AstSastClient.java b/src/main/java/com/cx/restclient/ast/AstSastClient.java new file mode 100644 index 00000000..236733b9 --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/AstSastClient.java @@ -0,0 +1,511 @@ +package com.cx.restclient.ast; + +import com.cx.restclient.ast.dto.common.*; +import com.cx.restclient.ast.dto.sast.AstSastConfig; +import com.cx.restclient.ast.dto.sast.AstSastResults; +import com.cx.restclient.ast.dto.sast.SastScanConfigValue; +import com.cx.restclient.ast.dto.sast.report.*; +import com.cx.restclient.common.Scanner; +import com.cx.restclient.common.UrlUtils; +import com.cx.restclient.configuration.CxScanConfig; +import com.cx.restclient.dto.*; +import com.cx.restclient.dto.scansummary.Severity; +import com.cx.restclient.exception.CxClientException; +import com.cx.restclient.exception.CxHTTPClientException; +import com.cx.restclient.httpClient.CxHttpClient; +import com.cx.restclient.httpClient.utils.ContentType; +import com.cx.restclient.osa.dto.ClientType; +import com.cx.restclient.sast.utils.State; +import com.cx.restclient.sast.utils.zip.CxZipUtils; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.lang3.EnumUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.*; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.ByteArrayEntity; +import org.apache.http.message.BasicNameValuePair; +import org.slf4j.Logger; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URLEncoder; +import java.util.*; +import java.util.stream.Collectors; + +public class AstSastClient extends AstClient implements Scanner { + private static final String ENGINE_TYPE_FOR_API = "sast"; + private static final String REF_TYPE_BRANCH = "branch"; + private static final String SUMMARY_PATH = properties.get("astSast.scanSummary"); + private static final String SCAN_RESULTS_PATH = properties.get("astSast.scanResults"); + private static final String AUTH_PATH = properties.get("astSast.authentication"); + private static final String WEB_PROJECT_PATH = properties.get("astSast.webProject"); + private static final String URL_PARSING_EXCEPTION = "URL parsing exception."; + private static final String DESCRIPTIONS_PATH = properties.get("astSast.descriptionPath"); + + private static final int DEFAULT_PAGE_SIZE = 1000; + private static final int NO_FINDINGS_CODE = 4004; + + private static final ObjectMapper objectMapper = new ObjectMapper(); + private static final String API_VERSION = "*/*; version=0.1"; + private static final String SCAN_ID_PARAM_NAME = "scan-id"; + private static final String OFFSET_PARAM_NAME = "offset"; + private static final String LIMIT_PARAM_NAME = "limit"; + private static final String ID_PARAM_NAME = "ids"; + private static final int URL_MAX_CHAR_SIZE = 1490; + + private String scanId; + + public AstSastClient(CxScanConfig config, Logger log) { + super(config, log); + + AstSastConfig astConfig = this.config.getAstSastConfig(); + validate(astConfig); + + // Make sure we won't get URLs like "http://example.com//api/scans". + String normalizedUrl = StringUtils.stripEnd(astConfig.getApiUrl(), "/"); + + httpClient = createHttpClient(normalizedUrl); + httpClient.addCustomHeader(HttpHeaders.ACCEPT, API_VERSION); + } + + @Override + public Results init() { + log.debug("Initializing {} client.", getScannerDisplayName()); + AstSastResults astResults = new AstSastResults(); + try { + ClientType clientType = getClientType(); + LoginSettings settings = getLoginSettings(clientType); + httpClient.login(settings); + } catch (Exception e) { + super.handleInitError(e, astResults); + } + return astResults; + } + + private LoginSettings getLoginSettings(ClientType clientType) throws MalformedURLException { + String authUrl = UrlUtils.parseURLToString(config.getAstSastConfig().getApiUrl(), AUTH_PATH); + return LoginSettings.builder() + .accessControlBaseUrl(authUrl) + .clientTypeForPasswordAuth(clientType) + .build(); + } + + private ClientType getClientType() { + AstSastConfig astConfig = config.getAstSastConfig(); + return ClientType.builder() + .clientId(astConfig.getClientId()) + .clientSecret(astConfig.getClientSecret()) + .scopes("ast-api") + .grantType("client_credentials") + .build(); + } + + @Override + protected String getScannerDisplayName() { + return ScannerType.AST_SAST.getDisplayName(); + } + + @Override + protected void uploadArchive(byte[] source, String uploadUrl) throws IOException { + log.info("Uploading the zipped data."); + + HttpEntity request = new ByteArrayEntity(source); + String baseAstUri = httpClient.getRootUri(); + httpClient.setRootUri(uploadUrl); + + try { + // Relative path is empty, because we use the whole upload URL as the base URL for the HTTP client. + // Content type is empty, because the server at uploadUrl throws an error if Content-Type is non-empty. + httpClient.putRequest("", "", request, JsonNode.class, HttpStatus.SC_OK, "upload ZIP file"); + } + finally { + httpClient.setRootUri(baseAstUri); + } + } + + @Override + public Results initiateScan() { + log.info("----------------------------------- Initiating {} Scan:------------------------------------", + getScannerDisplayName()); + + AstSastResults astResults = new AstSastResults(); + scanId = null; + + AstSastConfig astConfig = config.getAstSastConfig(); + try { + SourceLocationType locationType = astConfig.getSourceLocationType(); + HttpResponse response; + if (locationType == SourceLocationType.REMOTE_REPOSITORY) { + response = submitSourcesFromRemoteRepo(astConfig, config.getProjectName()); + } else { + + response = submitAllSourcesFromLocalDir(config.getProjectName(), astConfig.getZipFilePath()); + } + scanId = extractScanIdFrom(response); + astResults.setScanId(scanId); + } catch (Exception e) { + log.error(e.getMessage()); + setState(State.FAILED); + astResults.setException(new CxClientException("Error creating scan.", e)); + } + return astResults; + } + + protected HttpResponse submitAllSourcesFromLocalDir(String projectId, String zipFilePath) throws IOException { + log.info("Using local directory flow."); + + PathFilter filter = new PathFilter("", "", log); + String sourceDir = config.getSourceDir(); + byte[] zipFile = CxZipUtils.getZippedSources(config, filter, sourceDir, log); + + return initiateScanForUpload(projectId, zipFile, config.getAstSastConfig()); + } + + @Override + protected ScanConfig getScanConfig() { + String presetName = config.getAstSastConfig().getPresetName(); + if (StringUtils.isEmpty(presetName)) { + throw new CxClientException("Scan preset must be specified."); + } + + String isIncremental = Boolean.toString(config.getAstSastConfig().isIncremental()); + ScanConfigValue configValue = SastScanConfigValue.builder() + .incremental(isIncremental) + .presetName(presetName) + .build(); + + return ScanConfig.builder() + .type(ENGINE_TYPE_FOR_API) + .value(configValue) + .build(); + } + + @Override + protected HandlerRef getBranchToScan(RemoteRepositoryInfo repoInfo) { + // We need to return this object even if no branch is specified in repoInfo. + return HandlerRef.builder() + .type(REF_TYPE_BRANCH) + .value(repoInfo.getBranch()) + .build(); + } + + @Override + public Results waitForScanResults() { + AstSastResults result; + try { + waitForScanToFinish(scanId); + result = retrieveScanResults(); + } catch (CxClientException e) { + log.error(e.getMessage()); + result = new AstSastResults(); + result.setException(e); + } + return result; + } + + private AstSastResults retrieveScanResults() { + try { + AstSastResults result = new AstSastResults(); + result.setScanId(scanId); + + AstSastSummaryResults scanSummary = getSummary(); + result.setSummary(scanSummary); + + List findings = getFindings(); + result.setFindings(findings); + + String projectLink = getWebReportLink(config.getAstSastConfig().getWebAppUrl()); + result.setWebReportLink(projectLink); + + return result; + } catch (IOException e) { + String message = String.format("Error getting %s scan results.", getScannerDisplayName()); + throw new CxClientException(message, e); + } + } + + @Override + protected String getWebReportPath() throws UnsupportedEncodingException { + return String.format(WEB_PROJECT_PATH, + URLEncoder.encode(config.getProjectName(), ENCODING)); + } + + private AstSastSummaryResults getSummary() { + AstSastSummaryResults result = new AstSastSummaryResults(); + + String summaryUrl = getRelativeSummaryUrl(); + SummaryResponse summaryResponse = getSummaryResponse(summaryUrl); + + SingleScanSummary nativeSummary = getNativeSummary(summaryResponse); + setFindingCountsPerSeverity(nativeSummary.getSeverityCounters(), result); + + result.setStatusCounters(nativeSummary.getStatusCounters()); + result.setTotalCounter(nativeSummary.getTotalCounter()); + + return result; + } + + private List getFindings() throws IOException { + int offset = 0; + int limit = config.getAstSastConfig().getResultsPageSize(); + if (limit <= 0) { + limit = DEFAULT_PAGE_SIZE; + } + + List allFindings = new ArrayList<>(); + while (true) { + String relativeUrl = getRelativeResultsUrl(offset, limit); + ScanResultsResponse response = getScanResultsResponse(relativeUrl); + List findingsFromResponse = response.getResults(); + allFindings.addAll(findingsFromResponse); + offset += findingsFromResponse.size(); + if (offset >= response.getTotalCount()) { + break; + } + } + + log.info(String.format("Total findings: %d", allFindings.size())); + + + try { + populateAdditionalFields(allFindings); + } catch (CxClientException e) { + log.error(e.getMessage()); + } + + return allFindings; + } + + private void populateAdditionalFields(List allFindings) throws IOException { + + final Map allQueryDescriptionMap = new HashMap<>(); + + Set queryIDs = allFindings.stream().map(finding -> finding.getQueryID()).collect(Collectors.toSet()); + + while (queryIDs.size() > 0) { + Set processedQueryIds = new HashSet(); + List queryDescriptionList = processQueryIDs(queryIDs, processedQueryIds); + + allQueryDescriptionMap.putAll( + queryDescriptionList.stream().collect(Collectors.toMap(QueryDescription::getQueryId, queryDescription -> queryDescription))); + + queryIDs.removeAll(processedQueryIds); + } + + log.info(String.format("QueryIds with descriptions size: {} ", allQueryDescriptionMap.size())); + + allFindings.stream().forEach(finding -> { + String queryId = finding.getQueryID(); + QueryDescription query = allQueryDescriptionMap.get(queryId); + finding.setDescription(query.getResultDescription()); + }); + + + } + + private String prepareURL(Set ids, Set processedIds) { + try { + int lengthOtherParams = new URIBuilder().setPath(DESCRIPTIONS_PATH).setParameter(SCAN_ID_PARAM_NAME, scanId) + .build() + .toString().length(); + + URIBuilder uriBuilder = new URIBuilder(); + uriBuilder.setPath(DESCRIPTIONS_PATH); + + int idsAllowedLength = URL_MAX_CHAR_SIZE - lengthOtherParams; + + List nameValues = new LinkedList<>(); + + for (String id : ids) { + idsAllowedLength = idsAllowedLength - ID_PARAM_NAME.length() - 2 - id.length(); + if (idsAllowedLength > 0) { + processedIds.add(id); + nameValues.add(new BasicNameValuePair(ID_PARAM_NAME, id)); + } + } + + uriBuilder.setParameters(nameValues); + String result = uriBuilder.setParameter(SCAN_ID_PARAM_NAME, scanId) + .build() + .toString(); + + + log.debug(String.format("Getting descriptions from %s", result)); + + return result; + } catch (URISyntaxException e) { + throw new CxClientException(URL_PARSING_EXCEPTION, e); + } + } + + private String getRelativeResultsUrl(int offset, int limit) { + try { + String result = new URIBuilder() + .setPath(SCAN_RESULTS_PATH) + .setParameter(SCAN_ID_PARAM_NAME, scanId) + .setParameter(OFFSET_PARAM_NAME, Integer.toString(offset)) + .setParameter(LIMIT_PARAM_NAME, Integer.toString(limit)) + .build() + .toString(); + + if (log.isDebugEnabled()) { + log.debug(String.format("Getting findings from %s", result)); + } + + return result; + } catch (URISyntaxException e) { + throw new CxClientException(URL_PARSING_EXCEPTION, e); + } + } + + private List processQueryIDs(Set ids, Set processedIds) throws IOException { + + String relativeUrl = prepareURL(ids, processedIds); + + List result = (List) httpClient.getRequest(relativeUrl, + ContentType.CONTENT_TYPE_APPLICATION_JSON, + QueryDescription.class, + HttpStatus.SC_OK, + "retrieving queries description", + true); + + return result; + } + + private ScanResultsResponse getScanResultsResponse(String relativeUrl) throws IOException { + return httpClient.getRequest(relativeUrl, + ContentType.CONTENT_TYPE_APPLICATION_JSON, + ScanResultsResponse.class, + HttpStatus.SC_OK, + "retrieving scan results", + false); + } + + private SummaryResponse getSummaryResponse(String relativeUrl) { + SummaryResponse result; + try { + result = httpClient.getRequest(relativeUrl, + ContentType.CONTENT_TYPE_APPLICATION_JSON, + SummaryResponse.class, + HttpStatus.SC_OK, + "retrieving scan summary", + false); + } catch (Exception e) { + result = getEmptySummaryIfApplicable(e); + } + return result; + } + + private SummaryResponse getEmptySummaryIfApplicable(Exception e) { + SummaryResponse result; + if (noFindingsWereDetected(e)) { + result = new SummaryResponse(); + result.getScansSummaries().add(new SingleScanSummary()); + } else { + throw new CxClientException("Error getting scan summary.", e); + } + return result; + } + + /** + * When no findings are detected, AST-SAST API returns the 404 status with a specific + * error code, which is quite awkward. + * Response example: {"code":4004,"message":"can't find all the provided scan ids","data":null} + * + * @return true: scan completed successfully and the result contains no findings (normal flow). + * false: some other error has occurred (error flow). + */ + private boolean noFindingsWereDetected(Exception e) { + boolean result = false; + if (e instanceof CxHTTPClientException) { + CxHTTPClientException httpException = (CxHTTPClientException) e; + if (httpException.getStatusCode() == HttpStatus.SC_NOT_FOUND && + StringUtils.isNotEmpty(httpException.getResponseBody())) { + try { + JsonNode body = objectMapper.readTree(httpException.getResponseBody()); + result = (body.get("code").asInt() == NO_FINDINGS_CODE); + } catch (Exception parsingException) { + log.warn("Error parsing the 'Not found' response.", parsingException); + } + } + } + return result; + } + + + private String getRelativeSummaryUrl() { + try { + String result = new URIBuilder() + .setPath(SUMMARY_PATH) + .setParameter("scan-ids", scanId) + .build() + .toString(); + + if (log.isDebugEnabled()) { + log.debug(String.format("Getting summary from %s", result)); + } + + return result; + } catch (URISyntaxException e) { + throw new CxClientException(URL_PARSING_EXCEPTION, e); + } + } + + private static void setFindingCountsPerSeverity(List nativeCounters, AstSastSummaryResults target) { + if (nativeCounters == null) { + return; + } + + for (SeverityCounter counter : nativeCounters) { + Severity parsedSeverity = EnumUtils.getEnum(Severity.class, counter.getSeverity()); + int value = counter.getCounter(); + if (parsedSeverity != null) { + if (parsedSeverity == Severity.HIGH) { + target.setHighVulnerabilityCount(value); + } else if (parsedSeverity == Severity.MEDIUM) { + target.setMediumVulnerabilityCount(value); + } else if (parsedSeverity == Severity.LOW) { + target.setLowVulnerabilityCount(value); + } + } + } + } + + private static SingleScanSummary getNativeSummary(SummaryResponse summaryResponse) { + return Optional.ofNullable(summaryResponse).map(SummaryResponse::getScansSummaries) + // We are sending a single scan ID in the request and therefore expect exactly 1 scan summary. + .filter(scanSummaries -> scanSummaries.size() == 1) + .map(scanSummaries -> scanSummaries.get(0)) + .orElseThrow(() -> new CxClientException("Invalid summary response.")); + } + + @Override + public Results getLatestScanResults() { + log.error("Unsupported Operation."); + AstSastResults result = new AstSastResults(); + result.setException(new CxClientException(new UnsupportedOperationException())); + return result; + } + + @Override + public void close() { + Optional.ofNullable(httpClient).ifPresent(CxHttpClient::close); + } + + private void validate(ASTConfig astSastConfig) { + log.debug("Validating config."); + String error = null; + if (astSastConfig == null) { + error = "%s config must be provided."; + } else if (StringUtils.isBlank(astSastConfig.getApiUrl())) { + error = "%s API URL must be provided."; + } + + if (error != null) { + throw new IllegalArgumentException(String.format(error, getScannerDisplayName())); + } + } +} diff --git a/src/main/java/com/cx/restclient/ast/AstScaClient.java b/src/main/java/com/cx/restclient/ast/AstScaClient.java new file mode 100644 index 00000000..470fd38a --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/AstScaClient.java @@ -0,0 +1,864 @@ +package com.cx.restclient.ast; + +import com.cx.restclient.ast.dto.common.HandlerRef; +import com.cx.restclient.ast.dto.common.RemoteRepositoryInfo; +import com.cx.restclient.ast.dto.common.ScanConfig; +import com.cx.restclient.ast.dto.common.ScanConfigValue; +import com.cx.restclient.ast.dto.sca.*; +import com.cx.restclient.ast.dto.sca.report.AstScaSummaryResults; +import com.cx.restclient.ast.dto.sca.report.Finding; +import com.cx.restclient.ast.dto.sca.report.Package; +import com.cx.restclient.ast.dto.sca.report.PolicyEvaluation; +import com.cx.restclient.common.CxPARAM; +import com.cx.restclient.common.Scanner; +import com.cx.restclient.common.UrlUtils; +import com.cx.restclient.configuration.CxScanConfig; +import com.cx.restclient.dto.*; +import com.cx.restclient.exception.CxClientException; +import com.cx.restclient.exception.CxHTTPClientException; +import com.cx.restclient.httpClient.CxHttpClient; +import com.cx.restclient.httpClient.utils.ContentType; +import com.cx.restclient.httpClient.utils.HttpClientHelper; +import com.cx.restclient.osa.dto.ClientType; +import com.cx.restclient.osa.utils.OSAUtils; +import com.cx.restclient.sast.utils.State; +import com.cx.restclient.sast.utils.zip.CxZipUtils; +import com.cx.restclient.sast.utils.zip.NewCxZipFile; +import com.cx.restclient.sast.utils.zip.Zipper; +import com.cx.restclient.sca.dto.CxSCAResolvingConfiguration; +import com.cx.restclient.sca.utils.CxSCAFileSystemUtils; +import com.cx.restclient.sca.utils.fingerprints.CxSCAScanFingerprints; +import com.cx.restclient.sca.utils.fingerprints.FingerprintCollector; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.ByteArrayEntity; +import org.apache.http.entity.StringEntity; +import org.json.JSONObject; +import org.slf4j.Logger; + +import java.io.File; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URL; +import java.net.URLEncoder; +import java.nio.file.*; +import java.util.*; +import java.util.stream.Collectors; + +import static com.cx.restclient.sast.utils.SASTParam.MAX_ZIP_SIZE_BYTES; +import static com.cx.restclient.sast.utils.SASTParam.TEMP_FILE_NAME_TO_ZIP; + +/** + * SCA - Software Composition Analysis - is the successor of OSA. + */ +public class AstScaClient extends AstClient implements Scanner { + private static final String RISK_MANAGEMENT_API = properties.get("astSca.riskManagementApi"); + private static final String PROJECTS = RISK_MANAGEMENT_API + properties.get("astSca.projects"); + private static final String SUMMARY_REPORT = RISK_MANAGEMENT_API + properties.get("astSca.summaryReport"); + private static final String FINDINGS = RISK_MANAGEMENT_API + properties.get("astSca.findings"); + private static final String PACKAGES = RISK_MANAGEMENT_API + properties.get("astSca.packages"); + private static final String LATEST_SCAN = RISK_MANAGEMENT_API + properties.get("astSca.latestScan"); + private static final String WEB_REPORT = properties.get("astSca.webReport"); + private static final String RESOLVING_CONFIGURATION_API = properties.get("astSca.resolvingConfigurationApi"); + private static final String REPORTID_API = RISK_MANAGEMENT_API + properties.get("astSca.reportId"); + private static final String POLICY_MANAGEMENT_API = properties.get("astSca.policyManagementApi"); + private static final String POLICY_MANAGEMENT_EVALUATION_API = POLICY_MANAGEMENT_API + properties.get("astSca.policyManagementEvaliation"); + + private static final String REPORT_SCA_PACKAGES = "cxSCAPackages"; + private static final String REPORT_SCA_FINDINGS = "cxSCAVulnerabilities"; + private static final String REPORT_SCA_SUMMARY = "cxSCASummary"; + private static final String JSON_EXTENSION = ".json"; + + private static final String ENGINE_TYPE_FOR_API = "sca"; + + private static final String TENANT_HEADER_NAME = "Account-Name"; + + private static final ObjectMapper caseInsensitiveObjectMapper = new ObjectMapper() + // Ignore any fields that can be added to SCA API in the future. + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + // We need this feature to properly deserialize finding severity, + // e.g. "High" (in JSON) -> Severity.HIGH (in Java). + .enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS); + private final AstScaConfig astScaConfig; + + + private String projectId; + private String scanId; + private String reportId; + private final FingerprintCollector fingerprintCollector; + private CxSCAResolvingConfiguration resolvingConfiguration; + private static final String FINGERPRINT_FILE_NAME = ".cxsca.sig"; + private static final String SCA_CONFIG_FOLDER_NAME = ".cxsca.configurations"; + + public AstScaClient(CxScanConfig config, Logger log) { + super(config, log); + + this.astScaConfig = config.getAstScaConfig(); + validate(astScaConfig); + + httpClient = createHttpClient(astScaConfig.getApiUrl()); + this.resolvingConfiguration = null; + fingerprintCollector = new FingerprintCollector(log); + // Pass tenant name in a custom header. This will allow to get token from on-premise access control server + // and then use this token for SCA authentication in cloud. + httpClient.addCustomHeader(TENANT_HEADER_NAME, config.getAstScaConfig().getTenant()); + } + + @Override + protected String getScannerDisplayName() { + return ScannerType.AST_SCA.getDisplayName(); + } + + @Override + protected ScanConfig getScanConfig() { + + String sastProjectId = config.getAstScaConfig().getSastProjectId(); + String sastServerUrl = config.getAstScaConfig().getSastServerUrl(); + String sastUsername = config.getAstScaConfig().getSastUsername(); + String sastPassword = config.getAstScaConfig().getSastPassword(); + String sastProjectName = config.getAstScaConfig().getSastProjectName(); + + Map envVariables = config.getAstScaConfig().getEnvVariables(); + JSONObject envJsonString = new JSONObject(envVariables); + + ScanConfigValue configValue = ScaScanConfigValue.builder() + .environmentVariables(envJsonString.toString()) + .sastProjectId(sastProjectId) + .sastServerUrl(sastServerUrl) + .sastUsername(sastUsername) + .sastPassword(sastPassword) + .sastProjectName(sastProjectName) + .build(); + + return ScanConfig.builder() + .type(ENGINE_TYPE_FOR_API) + .value(configValue) + .build(); + + } + + @Override + protected HandlerRef getBranchToScan(RemoteRepositoryInfo repoInfo) { + if (StringUtils.isNotEmpty(repoInfo.getBranch())) { + // If we pass the branch to start scan API, the API will return an error: + // "Git references (branch, commit ID, etc.) are not yet supported." + // + // We can't just ignore the branch, because it will lead to confusion. + String message = String.format("Branch specification is not yet supported by %s.", getScannerDisplayName()); + throw new CxClientException(message); + } + return null; + } + + /** + * Transforms the repo URL if credentials are specified in repoInfo. + */ + @Override + protected URL getEffectiveRepoUrl(RemoteRepositoryInfo repoInfo) { + URL result; + URL initialUrl = repoInfo.getUrl(); + + // Otherwise we may get something like "https://mytoken:null@github.com". + String username = StringUtils.defaultString(repoInfo.getUsername()); + String password = StringUtils.defaultString(repoInfo.getPassword()); + + try { + if (StringUtils.isNotEmpty(username) || StringUtils.isNotEmpty(password)) { + log.info("Adding credentials as the userinfo part of the URL, because {} only supports this kind of authentication.", + getScannerDisplayName()); + + result = new URIBuilder(initialUrl.toURI()) + .setUserInfo(username, password) + .build() + .toURL(); + } else { + result = repoInfo.getUrl(); + } + } catch (Exception e) { + throw new CxClientException("Error getting effective repo URL."); + } + return result; + } + + @Override + public Results init() { + log.debug("Initializing {} client.", getScannerDisplayName()); + AstScaResults scaResults = new AstScaResults(); + try { + login(); + } catch (Exception e) { + super.handleInitError(e, scaResults); + } + return scaResults; + } + + public CxSCAResolvingConfiguration getCxSCAResolvingConfigurationForProject(String projectId) throws IOException { + log.info("Resolving configuration for project: {}", projectId); + String path = String.format(RESOLVING_CONFIGURATION_API, URLEncoder.encode(projectId, ENCODING)); + + return httpClient.getRequest(path, + ContentType.CONTENT_TYPE_APPLICATION_JSON, + CxSCAResolvingConfiguration.class, + HttpStatus.SC_OK, + "get CxSCA resolving configuration", + false); + } + + /** + * Waits for SCA scan to finish, then gets scan results. + * + * @throws CxClientException in case of a network error, scan failure or when scan is aborted by timeout. + */ + @Override + public Results waitForScanResults() { + AstScaResults scaResults; + try { + waitForScanToFinish(scanId); + scaResults = tryGetScanResults().orElseThrow(() -> new CxClientException("Unable to get scan results: scan not found.")); + if (config.getScaJsonReport() != null) { + OSAUtils.writeJsonToFile(REPORT_SCA_FINDINGS + JSON_EXTENSION, scaResults.getFindings(), config.getReportsDir(), config.getOsaGenerateJsonReport(), log); + OSAUtils.writeJsonToFile(REPORT_SCA_PACKAGES + JSON_EXTENSION, scaResults.getPackages(), config.getReportsDir(), config.getOsaGenerateJsonReport(), log); + OSAUtils.writeJsonToFile(REPORT_SCA_SUMMARY + JSON_EXTENSION, scaResults.getSummary(), config.getReportsDir(), config.getOsaGenerateJsonReport(), log); + } + } catch (CxClientException e) { + log.error(e.getMessage()); + scaResults = new AstScaResults(); + scaResults.setException(e); + } + return scaResults; + } + + @Override + protected void uploadArchive(byte[] source, String uploadUrl) throws IOException { + log.info("Uploading the zipped data."); + CxHttpClient uploader = null; + HttpEntity request = new ByteArrayEntity(source); + + try { + uploader = createHttpClient(uploadUrl); + + // Relative path is empty, because we use the whole upload URL as the base URL for the HTTP client. + // Content type is empty, because the server at uploadUrl throws an error if Content-Type is non-empty. + uploader.putRequest("", "", request, JsonNode.class, HttpStatus.SC_OK, "upload ZIP file"); + }finally { + Optional.ofNullable(uploader).ifPresent(CxHttpClient::close); + } + + } + + @Override + public Results initiateScan() { + log.info("----------------------------------- Initiating {} Scan:------------------------------------", + getScannerDisplayName()); + AstScaResults scaResults = new AstScaResults(); + scanId = null; + projectId = null; + try { + AstScaConfig scaConfig = config.getAstScaConfig(); + SourceLocationType locationType = scaConfig.getSourceLocationType(); + HttpResponse response; + + projectId = resolveRiskManagementProject(); + boolean isManifestAndFingerprintsOnly = !config.getAstScaConfig().isIncludeSources(); + if (isManifestAndFingerprintsOnly) { + this.resolvingConfiguration = getCxSCAResolvingConfigurationForProject(this.projectId); + log.info("Got the following manifest patterns {}", this.resolvingConfiguration.getManifests()); + log.info("Got the following fingerprint patterns {}", this.resolvingConfiguration.getFingerprints()); + } + + if (locationType == SourceLocationType.REMOTE_REPOSITORY) { + response = submitSourcesFromRemoteRepo(scaConfig, projectId); + } else { + if (scaConfig.isIncludeSources()) { + response = submitAllSourcesFromLocalDir(projectId, astScaConfig.getZipFilePath()); + } else { + response = submitManifestsAndFingerprintsFromLocalDir(projectId); + } + } + this.scanId = extractScanIdFrom(response); + scaResults.setScanId(scanId); + } catch (Exception e) { + log.error("Error occurred while initiating scan.", e); + setState(State.FAILED); + scaResults.setException(new CxClientException("Error creating scan.", e)); + } + return scaResults; + } + + protected HttpResponse submitAllSourcesFromLocalDir(String projectId, String zipFilePath) throws IOException { + log.info("Using local directory flow."); + + PathFilter filter = new PathFilter(config.getOsaFolderExclusions(), config.getOsaFilterPattern(), log); + String sourceDir = config.getEffectiveSourceDirForDependencyScan(); + + Path configFileDestination = copyConfigFileToSourceDir(sourceDir); + + byte[] zipFile = CxZipUtils.getZippedSources(config, filter, sourceDir, log); + + + FileUtils.deleteDirectory(configFileDestination.toFile()); + + return initiateScanForUpload(projectId, zipFile, config.getAstScaConfig()); + } + + private HttpResponse submitManifestsAndFingerprintsFromLocalDir(String projectId) throws IOException { + log.info("Using manifest only and fingerprint flow"); + String sourceDir = config.getEffectiveSourceDirForDependencyScan(); + Path configFileDestination = copyConfigFileToSourceDir(sourceDir); + String additinalFilters = getAdditionalManifestFilters(configFileDestination); + String finalFilters = additinalFilters + getManifestsIncludePattern(); + PathFilter userFilter = new PathFilter(config.getOsaFolderExclusions(), config.getOsaFilterPattern(), log); + if (ArrayUtils.isNotEmpty(userFilter.getIncludes()) && !ArrayUtils.contains(userFilter.getIncludes(), "**")) { + userFilter.addToIncludes("**"); + } + Set scannedFileSet = new HashSet<>(Arrays.asList(CxSCAFileSystemUtils.scanAndGetIncludedFiles(sourceDir, userFilter))); + + PathFilter manifestIncludeFilter = new PathFilter(null,finalFilters , log); + if (manifestIncludeFilter.getIncludes().length == 0) { + throw new CxClientException(String.format("Using manifest only mode requires include filter. Resolving config does not have include patterns defined: %s", getManifestsIncludePattern())); + } + + List filesToZip = + Arrays.stream(CxSCAFileSystemUtils.scanAndGetIncludedFiles(sourceDir, manifestIncludeFilter)) + .filter(scannedFileSet::contains). + collect(Collectors.toList()); + + List filesToFingerprint = + Arrays.stream(CxSCAFileSystemUtils.scanAndGetIncludedFiles(sourceDir, + new PathFilter(null, getFingerprintsIncludePattern(), log))) + .filter(scannedFileSet::contains). + collect(Collectors.toList()); + + + CxSCAScanFingerprints fingerprints = fingerprintCollector.collectFingerprints(sourceDir, filesToFingerprint); + + File zipFile = zipDirectoryAndFingerprints(sourceDir, filesToZip, fingerprints); + + optionallyWriteFingerprintsToFile(fingerprints); + + + FileUtils.deleteDirectory(configFileDestination.toFile()); + + return initiateScanForUpload(projectId, FileUtils.readFileToByteArray(zipFile), astScaConfig); + } + /** + * + * This method gets the additional config file(from different package manager) manifest filters + * e.g. returns "settings.xml,npmrc"/" + **/ + + private String getAdditionalManifestFilters(Path configFileDestination) { + List configFilePaths = config.getAstScaConfig().getConfigFilePaths(); + String additionalFilters = ""; + if (configFilePaths != null) { + for (String configFileString : configFilePaths) { + if (StringUtils.isNotEmpty(configFileString)) { + if (configFileString.lastIndexOf("\\") != -1) + configFileString = configFileString.substring(configFileString.lastIndexOf("\\") + 1); + additionalFilters = additionalFilters.concat("**/" + configFileString + ","); + } + } + } + return additionalFilters; + } + + private Path copyConfigFileToSourceDir(String sourceDir) throws IOException { + + Path configFileDestination = Paths.get(""); + log.info("Source Directory : " + sourceDir); + List configFilePaths = config.getAstScaConfig().getConfigFilePaths(); + + if(configFilePaths != null) { + for(String configFileString : configFilePaths) { + + if (StringUtils.isNotEmpty(configFileString)) { + String fileSystemSeparator = FileSystems.getDefault().getSeparator(); + Path configFilePath = CxSCAFileSystemUtils.checkIfFileExists(sourceDir, configFileString, fileSystemSeparator, log); + + if (configFilePath != null) { + configFileDestination = Paths.get(sourceDir, fileSystemSeparator, SCA_CONFIG_FOLDER_NAME); + + if (Files.notExists(configFileDestination)) { + Path destDir = Files.createDirectory(configFileDestination); + Files.copy(configFilePath, destDir.resolve(configFilePath.getFileName()), StandardCopyOption.REPLACE_EXISTING); + log.info("Config file (" + configFilePath + ") copied to directory: " + configFileDestination); + + } else { + Path r = configFileDestination.resolve(configFilePath.getFileName()); + Files.copy(configFilePath,r , StandardCopyOption.REPLACE_EXISTING); + log.info("Config file (" + configFilePath + ") copied to directory: " + configFileDestination); + } + } + } + } + } + return configFileDestination; + } + + + + private File zipDirectoryAndFingerprints(String sourceDir, List paths, CxSCAScanFingerprints fingerprints) throws IOException { + File result = config.getZipFile(); + if (result != null) { + return result; + } + File tempFile = getZipFile(); + log.debug("Collecting files to zip archive: {}", tempFile.getAbsolutePath()); + long maxZipSizeBytes = config.getMaxZipSize() != null ? config.getMaxZipSize() * 1024 * 1024 : MAX_ZIP_SIZE_BYTES; + + try (NewCxZipFile zipper = new NewCxZipFile(tempFile, maxZipSizeBytes, log)) { + zipper.addMultipleFilesToArchive(new File(sourceDir), paths); + if (zipper.getFileCount() == 0 && fingerprints.getFingerprints().isEmpty()) { + throw handleFileDeletion(tempFile); + } + if (!fingerprints.getFingerprints().isEmpty()) { + zipper.zipContentAsFile(FINGERPRINT_FILE_NAME, FingerprintCollector.getFingerprintsAsJsonString(fingerprints).getBytes()); + } else { + log.debug("No supported fingerprints found to zip"); + } + + log.debug("The sources were zipped to {}", tempFile.getAbsolutePath()); + return tempFile; + } catch (Zipper.MaxZipSizeReached e) { + throw handleFileDeletion(tempFile, new IOException("Reached maximum upload size limit of " + FileUtils.byteCountToDisplaySize(maxZipSizeBytes))); + } catch (IOException ioException) { + throw handleFileDeletion(tempFile, ioException); + } + } + + private CxClientException handleFileDeletion(File file, IOException ioException) { + try { + Files.delete(file.toPath()); + } catch (IOException e) { + return new CxClientException(e); + } + + return new CxClientException(ioException); + + } + + private CxClientException handleFileDeletion(File file) { + try { + Files.delete(file.toPath()); + } catch (IOException e) { + return new CxClientException(e); + } + + return new CxClientException("No files found to zip and no supported fingerprints found"); + } + + private String getFingerprintsIncludePattern() { + if (StringUtils.isNotEmpty(astScaConfig.getFingerprintsIncludePattern())) { + return astScaConfig.getFingerprintsIncludePattern(); + } + + return resolvingConfiguration.getFingerprintsIncludePattern(); + } + + private String getManifestsIncludePattern() { + if (StringUtils.isNotEmpty(astScaConfig.getManifestsIncludePattern())) { + return astScaConfig.getManifestsIncludePattern(); + } + + return resolvingConfiguration.getManifestsIncludePattern(); + } + + private File getZipFile() throws IOException { + if (StringUtils.isNotEmpty(astScaConfig.getZipFilePath())) { + return new File(astScaConfig.getZipFilePath()); + } + return File.createTempFile(TEMP_FILE_NAME_TO_ZIP, ".bin"); + } + + private void optionallyWriteFingerprintsToFile(CxSCAScanFingerprints fingerprints) { + if (StringUtils.isNotEmpty(astScaConfig.getFingerprintFilePath())) { + try { + fingerprintCollector.writeScanFingerprintsFile(fingerprints, astScaConfig.getFingerprintFilePath()); + } catch (IOException ioException) { + log.error(String.format("Failed writing fingerprint file to %s", astScaConfig.getFingerprintFilePath()), ioException); + } + } + } + + /** + * Gets latest scan results using {@link CxScanConfig#getProjectName()} for the current config. + * + * @return results of the latest successful scan for a project, if present; null - otherwise. + */ + @Override + public Results getLatestScanResults() { + AstScaResults result = new AstScaResults(); + try { + log.info("Getting latest scan results."); + projectId = getRiskManagementProjectId(config.getProjectName()); + scanId = getLatestScanId(projectId); + result = tryGetScanResults().orElse(null); + } catch (Exception e) { + log.error(e.getMessage()); + result.setException(new CxClientException("Error getting latest scan results.", e)); + } + return result; + } + + private Optional tryGetScanResults() { + AstScaResults result = null; + if (StringUtils.isNotEmpty(scanId)) { + result = getScanResults(); + } else { + log.info("Unable to get scan results"); + } + return Optional.ofNullable(result); + } + + private String getLatestScanId(String projectId) throws IOException { + String result = null; + if (StringUtils.isNotEmpty(projectId)) { + log.debug("Getting latest scan ID for project ID: {}", projectId); + String path = String.format(LATEST_SCAN, URLEncoder.encode(projectId, ENCODING)); + JsonNode response = httpClient.getRequest(path, + ContentType.CONTENT_TYPE_APPLICATION_JSON, + ArrayNode.class, + HttpStatus.SC_OK, + "scan ID by project ID", + false); + + result = Optional.ofNullable(response) + // 'riskReportId' is in fact scanId, but the name is kept for backward compatibility. + .map(resp -> resp.at("/0/riskReportId").textValue()) + .orElse(null); + } + String message = (result == null ? "Scan not found" : String.format("Scan ID: %s", result)); + log.info(message); + return result; + } + + + private void printWebReportLink(AstScaResults scaResult) { + if (!StringUtils.isEmpty(scaResult.getWebReportLink())) { + log.info("{} scan results location: {}", getScannerDisplayName(), scaResult.getWebReportLink()); + } + } + + void testConnection() throws IOException { + // The calls below allow to check both access control and API connectivity. + login(); + getRiskManagementProjects(); + } + + public void login() throws IOException { + log.info("Logging into {}", getScannerDisplayName()); + AstScaConfig scaConfig = config.getAstScaConfig(); + + String acUrl = scaConfig.getAccessControlUrl(); + LoginSettings settings = LoginSettings.builder() + .accessControlBaseUrl(UrlUtils.parseURLToString(acUrl, CxPARAM.AUTHENTICATION)) + .username(scaConfig.getUsername()) + .password(scaConfig.getPassword()) + .tenant(scaConfig.getTenant()) + .build(); + + ClientTypeResolver resolver = new ClientTypeResolver(config); + ClientType clientType = resolver.determineClientType(acUrl); + settings.setClientTypeForPasswordAuth(clientType); + + httpClient.login(settings); + } + + public void close() { + if (httpClient != null) { + httpClient.close(); + } + } + + /** + * The following config properties are used: + * astScaConfig + * proxyConfig + * cxOrigin + * disableCertificateValidation + */ + public void testScaConnection() { + try { + testConnection(); + } catch (IOException e) { + throw new CxClientException(e); + } + } + + private String resolveRiskManagementProject() throws IOException { + String projectName = config.getProjectName(); + String assignedTeam = config.getTeamPath(); + log.info("Getting project by name: '{}'", projectName); + String resolvedProjectId = getRiskManagementProjectId(projectName); + if (resolvedProjectId == null) { + log.info("Project not found, creating a new one."); + resolvedProjectId = createRiskManagementProject(projectName, assignedTeam); + log.info("Created a project with ID {}", resolvedProjectId); + } else { + log.info("Project already exists with ID {}", resolvedProjectId); + } + return resolvedProjectId; + } + + private String getRiskManagementProjectId(String projectName) throws IOException { + log.info("Getting project ID by name: '{}'", projectName); + + if (StringUtils.isEmpty(projectName)) { + throw new CxClientException("Non-empty project name must be provided."); + } + + Project project = sendGetProjectRequest(projectName); + + String result = Optional.ofNullable(project) + .map(Project::getId) + .orElse(null); + + String message = (result == null ? "Project not found" : String.format("Project ID: %s", result)); + log.info(message); + + return result; + } + + private Project sendGetProjectRequest(String projectName) throws IOException { + Project result; + try { + String getProjectByName = String.format("%s?name=%s", PROJECTS, URLEncoder.encode(projectName, ENCODING)); + result = httpClient.getRequest(getProjectByName, + ContentType.CONTENT_TYPE_APPLICATION_JSON, + Project.class, + HttpStatus.SC_OK, + "CxSCA project ID by name", + false); + } catch (CxHTTPClientException e) { + if (e.getStatusCode() == HttpStatus.SC_NOT_FOUND) { + result = null; + } else { + throw e; + } + } + return result; + } + + private void getRiskManagementProjects() throws IOException { + httpClient.getRequest(PROJECTS, + ContentType.CONTENT_TYPE_APPLICATION_JSON, + Project.class, + HttpStatus.SC_OK, + "CxSCA projects", + true); + } + + private String createRiskManagementProject(String name, String assignedTeam) throws IOException { + CreateProjectRequest request = new CreateProjectRequest(); + request.setName(name); + if(!StringUtils.isEmpty(assignedTeam)) { + request.addAssignedTeams(assignedTeam); + log.info("Team name: "+assignedTeam); + } + + StringEntity entity = HttpClientHelper.convertToStringEntity(request); + + Project newProject = httpClient.postRequest(PROJECTS, + ContentType.CONTENT_TYPE_APPLICATION_JSON, + entity, + Project.class, + HttpStatus.SC_CREATED, + "create a project"); + + return newProject.getId(); + } + + private AstScaResults getScanResults() { + AstScaResults result; + log.debug("Getting results for scan ID {}", scanId); + try { + result = new AstScaResults(); + result.setScanId(this.scanId); + + reportId = getReportId(scanId); + result.setReportId(reportId); + + AstScaSummaryResults scanSummary = getSummaryReport(scanId); + result.setSummary(scanSummary); + printSummary(scanSummary, this.scanId); + + List findings = getFindings(scanId); + result.setFindings(findings); + + List packages = getPackages(scanId); + result.setPackages(packages); + + if(config.isEnablePolicyViolations()) { + List policyEvaluations = getPolicyEvaluation(reportId); + result.setPolicyEvaluations(policyEvaluations); + printPolicyEvaluations(policyEvaluations); + determinePolicyViolations(result); + } + + String reportLink = getWebReportLink(config.getAstScaConfig().getWebAppUrl()); + result.setWebReportLink(reportLink); + printWebReportLink(result); + result.setScaResultReady(true); + log.info("Retrieved SCA results successfully."); + } catch (IOException e) { + throw new CxClientException("Error retrieving CxSCA scan results.", e); + } + return result; + } + + @Override + protected String getWebReportPath() throws UnsupportedEncodingException { + return String.format(WEB_REPORT, + URLEncoder.encode(projectId, ENCODING), + URLEncoder.encode(scanId, ENCODING)); + } + + private AstScaSummaryResults getSummaryReport(String scanId) throws IOException { + log.debug("Getting summary report."); + + String path = String.format(SUMMARY_REPORT, URLEncoder.encode(scanId, ENCODING)); + + return httpClient.getRequest(path, + ContentType.CONTENT_TYPE_APPLICATION_JSON, + AstScaSummaryResults.class, + HttpStatus.SC_OK, + "CxSCA report summary", + false); + } + + private List getFindings(String scanId) throws IOException { + log.debug("Getting findings."); + + String path = String.format(FINDINGS, URLEncoder.encode(scanId, ENCODING)); + + ArrayNode responseJson = httpClient.getRequest(path, + ContentType.CONTENT_TYPE_APPLICATION_JSON, + ArrayNode.class, + HttpStatus.SC_OK, + "CxSCA findings", + false); + + Finding[] findings = caseInsensitiveObjectMapper.treeToValue(responseJson, Finding[].class); + + return Arrays.asList(findings); + } + + private List getPackages(String scanId) throws IOException { + log.debug("Getting packages."); + + String path = String.format(PACKAGES, URLEncoder.encode(scanId, ENCODING)); + + return (List) httpClient.getRequest(path, + ContentType.CONTENT_TYPE_APPLICATION_JSON, + Package.class, + HttpStatus.SC_OK, + "CxSCA findings", + true); + } + + public String getReportId(String scanId) throws IOException { + log.debug("Getting report id."); + + String path = String.format(REPORTID_API, URLEncoder.encode(scanId, ENCODING)); + + String resultReportId = (String) httpClient.getRequest(path, + ContentType.CONTENT_TYPE_APPLICATION_JSON, + String.class, + HttpStatus.SC_OK, + "CxSCA Risk ReportId", + false); + return StringUtils.strip(resultReportId, "\""); + } + + public List getPolicyEvaluation(String reportId) throws IOException { + log.debug("Getting policy evaluation for the scan report id " + reportId + "."); + + String path = String.format(POLICY_MANAGEMENT_EVALUATION_API, URLEncoder.encode(reportId, ENCODING)); + + return (List) httpClient.getRequest(path, + ContentType.CONTENT_TYPE_APPLICATION_JSON, + PolicyEvaluation.class, + HttpStatus.SC_OK, + "CxSCA policy evaulation", + true); + } + + private void determinePolicyViolations(AstScaResults result) { + + result.getPolicyEvaluations().forEach((p)-> { + if(p.getIsViolated()) { + //its enough even one policy is violated + result.setPolicyViolated(true); + if(p.getActions().isBreakBuild()) + result.setBreakTheBuild(true); + } + } + ); + } + + private void printSummary(AstScaSummaryResults summary, String scanId) { + if (log.isInfoEnabled()) { + log.info("----CxSCA risk report summary----"); + log.info("Created on: {}", summary.getCreatedOn()); + log.info("Direct packages: {}", summary.getDirectPackages()); + log.info("High vulnerabilities: {}", summary.getHighVulnerabilityCount()); + log.info("Medium vulnerabilities: {}", summary.getMediumVulnerabilityCount()); + log.info("Low vulnerabilities: {}", summary.getLowVulnerabilityCount()); + log.info("Scan ID: {}", scanId); + log.info(String.format("Risk score: %.2f", summary.getRiskScore())); + log.info("Total packages: {}", summary.getTotalPackages()); + log.info("Total outdated packages: {}", summary.getTotalOutdatedPackages()); + } + } + + private void printPolicyEvaluations(List policyEvaulations) { + if (log.isInfoEnabled()) { + log.info("----CxSCA Policy Evaluation Results----"); + policyEvaulations.forEach((p)-> printPolicyEvaluation(p)); + log.info("---------------------------------------"); + } + } + + private void printPolicyEvaluation(PolicyEvaluation p) { + if (log.isInfoEnabled()) { + log.info(" Policy name: " + p.getName() + " | Violated:" + p.getIsViolated() + " | Policy Description: " + p.getDescription()); + + p.getRules().forEach((r)-> + log.info(" Rule name: " + r.getName() + " | Violated:" + r.getIsViolated()) + ); + + } + } + + private void validate(AstScaConfig config) { + String error = null; + if (config == null) { + error = "%s config must be provided."; + } else if (StringUtils.isEmpty(config.getApiUrl())) { + error = "%s API URL must be provided."; + } else if (StringUtils.isEmpty(config.getAccessControlUrl())) { + error = "%s access control URL must be provided."; + } else { + RemoteRepositoryInfo repoInfo = config.getRemoteRepositoryInfo(); + if (repoInfo == null && config.getSourceLocationType() == SourceLocationType.REMOTE_REPOSITORY) { + error = "%s remote repository info must be provided."; + } else if (repoInfo != null && StringUtils.isNotEmpty(repoInfo.getBranch())) { + error = "%s doesn't support specifying custom branches. It currently uses the default branch of a repo."; + } + } + + if (error != null) { + throw new IllegalArgumentException(String.format(error, getScannerDisplayName())); + } + } +} diff --git a/src/main/java/com/cx/restclient/ast/AstWaiter.java b/src/main/java/com/cx/restclient/ast/AstWaiter.java new file mode 100644 index 00000000..5dd0bcb0 --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/AstWaiter.java @@ -0,0 +1,138 @@ +package com.cx.restclient.ast; + +import com.cx.restclient.common.ShragaUtils; +import com.cx.restclient.configuration.CxScanConfig; +import com.cx.restclient.exception.CxClientException; +import com.cx.restclient.httpClient.CxHttpClient; +import com.cx.restclient.httpClient.utils.ContentType; +import com.cx.restclient.ast.dto.common.ScanInfoResponse; +import com.cx.restclient.ast.dto.common.ScanStatus; +import lombok.RequiredArgsConstructor; +import org.slf4j.Logger; +import org.apache.commons.lang3.EnumUtils; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.http.HttpStatus; +import org.awaitility.Awaitility; +import org.awaitility.core.ConditionTimeoutException; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.time.Duration; +import java.util.concurrent.atomic.AtomicInteger; + +import static com.cx.restclient.ast.AstClient.ENCODING; + +@RequiredArgsConstructor +public class AstWaiter { + private final CxHttpClient httpClient; + private final CxScanConfig config; + private final String scannerDisplayName; + private long startTimestampSec; + private final Logger log; + + public void waitForScanToFinish(String scanId) { + startTimestampSec = System.currentTimeMillis() / 1000; + Duration timeout = getTimeout(config); + Duration pollInterval = getPollInterval(config); + + int maxErrorCount = getMaxErrorCount(config); + AtomicInteger errorCounter = new AtomicInteger(); + + try { + String urlPath = String.format(AstClient.GET_SCAN, URLEncoder.encode(scanId, ENCODING)); + + Awaitility.await() + .atMost(timeout) + .pollDelay(Duration.ZERO) + .pollInterval(pollInterval) + .until(() -> scanIsCompleted(urlPath, errorCounter, maxErrorCount)); + + } catch (ConditionTimeoutException e) { + String message = String.format( + "Failed to perform %s scan. The scan has been automatically aborted: " + + "reached the user-specified timeout (%d minutes).", + scannerDisplayName, + timeout.toMinutes()); + throw new CxClientException(message); + } catch (UnsupportedEncodingException e) { + log.error("Unexpected error.", e); + } + } + + private static Duration getTimeout(CxScanConfig config) { + Integer rawTimeout = config.getOsaScanTimeoutInMinutes(); + final int DEFAULT_TIMEOUT = 30; + rawTimeout = rawTimeout != null && rawTimeout > 0 ? rawTimeout : DEFAULT_TIMEOUT; + return Duration.ofMinutes(rawTimeout); + } + + private static Duration getPollInterval(CxScanConfig config) { + int rawPollInterval = ObjectUtils.defaultIfNull(config.getOsaProgressInterval(), 20); + return Duration.ofSeconds(rawPollInterval); + } + + private static int getMaxErrorCount(CxScanConfig config) { + return ObjectUtils.defaultIfNull(config.getConnectionRetries(), 3); + } + + private boolean scanIsCompleted(String path, AtomicInteger errorCounter, int maxErrorCount) { + ScanInfoResponse response = null; + String errorMessage = null; + try { + String failedMessage = scannerDisplayName + " scan"; + response = httpClient.getRequest(path, ContentType.CONTENT_TYPE_APPLICATION_JSON, + ScanInfoResponse.class, HttpStatus.SC_OK, failedMessage, false); + } catch (Exception e) { + errorMessage = e.getMessage(); + } + + boolean completedSuccessfully = false; + if (response == null) { + // A network error is likely to have occurred -> retry. + countError(errorCounter, maxErrorCount, errorMessage); + } else { + ScanStatus status = extractScanStatusFrom(response); + completedSuccessfully = handleScanStatus(status); + } + + return completedSuccessfully; + } + + private boolean handleScanStatus(ScanStatus status) { + boolean completedSuccessfully = false; + if (status == ScanStatus.COMPLETED) { + completedSuccessfully = true; + } else if (status == ScanStatus.FAILED) { + // Scan has failed on the back end, no need to retry. + throw new CxClientException(String.format("Scan status is %s, aborting.", status)); + } else if (status == null) { + log.warn("Unknown status."); + } + return completedSuccessfully; + } + + private void countError(AtomicInteger errorCounter, int maxErrorCount, String message) { + int currentErrorCount = errorCounter.incrementAndGet(); + int triesLeft = maxErrorCount - currentErrorCount; + if (triesLeft < 0) { + String fullMessage = String.format("Maximum number of errors was reached (%d), aborting.", maxErrorCount); + throw new CxClientException(fullMessage); + } else { + String note = (triesLeft == 0 ? "last attempt" : String.format("tries left: %d", triesLeft)); + log.info(String.format("Failed to get status from %s with the message: %s. Retrying (%s)", + scannerDisplayName, + message, + note)); + } + } + + private ScanStatus extractScanStatusFrom(ScanInfoResponse response) { + String rawStatus = response.getStatus(); + String elapsedTimestamp = ShragaUtils.getTimestampSince(startTimestampSec); + log.info(String.format("Waiting for %s scan results. Elapsed time: %s. Status: %s.", + scannerDisplayName, + elapsedTimestamp, + rawStatus)); + return EnumUtils.getEnumIgnoreCase(ScanStatus.class, rawStatus); + } +} diff --git a/src/main/java/com/cx/restclient/ast/ClientTypeResolver.java b/src/main/java/com/cx/restclient/ast/ClientTypeResolver.java new file mode 100644 index 00000000..f2e0f77c --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/ClientTypeResolver.java @@ -0,0 +1,114 @@ +package com.cx.restclient.ast; + +import com.cx.restclient.configuration.CxScanConfig; +import com.cx.restclient.exception.CxClientException; +import com.cx.restclient.httpClient.CxHttpClient; +import com.cx.restclient.osa.dto.ClientType; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; + +import static com.cx.restclient.httpClient.utils.ContentType.CONTENT_TYPE_APPLICATION_JSON_V1; + +@Slf4j +public class ClientTypeResolver { + private static final String WELL_KNOWN_CONFIG_PATH = "identity/.well-known/openid-configuration"; + private static final String SCOPES_JSON_PROP = "scopes_supported"; + + private static final Set scopesForCloudAuth = new HashSet<>(Arrays.asList("sca_api", "offline_access")); + private static final Set scopesForOnPremAuth = new HashSet<>(Arrays.asList("sast_rest_api", "cxarm_api")); + + private static final ObjectMapper objectMapper = new ObjectMapper(); + + private CxHttpClient httpClient; + + private CxScanConfig config; + + public ClientTypeResolver(CxScanConfig config) { + this.config = config; + } + + /** + * Determines which scopes and client secret must be used for SCA login. + * + * @param accessControlServerBaseUrl used to determine scopes supported by this server. + * @return client settings for the provided AC server. + */ + public ClientType determineClientType(String accessControlServerBaseUrl) { + JsonNode response = getConfigResponse(accessControlServerBaseUrl); + Set supportedScopes = getSupportedScopes(response); + Set scopesToUse = getScopesForAuth(supportedScopes); + + String clientSecret = scopesToUse.equals(scopesForOnPremAuth) ? ClientType.RESOURCE_OWNER.getClientSecret() : ""; + + String scopesForRequest = String.join(" ", scopesToUse); + + return ClientType.builder().clientId(ClientType.RESOURCE_OWNER.getClientId()) + .scopes(scopesForRequest) + .clientSecret(clientSecret) + .build(); + } + + private Set getScopesForAuth(Set supportedScopes) { + Set result; + if (supportedScopes.containsAll(scopesForCloudAuth)) { + result = scopesForCloudAuth; + } else if (supportedScopes.containsAll(scopesForOnPremAuth)) { + result = scopesForOnPremAuth; + } else { + String message = String.format("Access control server doesn't support the necessary scopes (either %s or %s)." + + " It only supports the following scopes: %s.", + scopesForCloudAuth, + scopesForOnPremAuth, + supportedScopes); + + throw new CxClientException(message); + } + log.debug(String.format("Using scopes: %s", result)); + return result; + } + + private JsonNode getConfigResponse(String accessControlServerBaseUrl) { + try { + String res = getHttpClient(accessControlServerBaseUrl).getRequest(WELL_KNOWN_CONFIG_PATH, CONTENT_TYPE_APPLICATION_JSON_V1, String.class, 200, "Get openId configuration", false); + return objectMapper.readTree(res); + } catch (Exception e) { + throw new CxClientException("Error getting OpenID config response.", e); + } + } + + private CxHttpClient getHttpClient(String acBaseUrl) { + if (httpClient == null) { + httpClient = new CxHttpClient( + StringUtils.appendIfMissing(acBaseUrl, "/"), + config.getCxOrigin(), + config.getCxOriginUrl(), + config.isDisableCertificateValidation(), + config.isUseSSOLogin(), + config.getRefreshToken(), + config.isProxy(), + config.getProxyConfig(), + log, + config.getNTLM()); + } + return httpClient; + } + + private static Set getSupportedScopes(JsonNode response) { + Set result = null; + if (response != null) { + TypeReference> typeRef = new TypeReference>() { + }; + result = objectMapper.convertValue(response.get(SCOPES_JSON_PROP), typeRef); + } + return Optional.ofNullable(result).orElse(new HashSet<>()); + } + +} diff --git a/src/main/java/com/cx/restclient/ast/dto/common/ASTConfig.java b/src/main/java/com/cx/restclient/ast/dto/common/ASTConfig.java new file mode 100644 index 00000000..5ec1dac5 --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/common/ASTConfig.java @@ -0,0 +1,25 @@ +package com.cx.restclient.ast.dto.common; + +import com.cx.restclient.dto.SourceLocationType; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.SuperBuilder; + +import java.io.Serializable; + +@Getter +@Setter +@NoArgsConstructor +@SuperBuilder +public abstract class ASTConfig implements Serializable { + private String apiUrl; + private String webAppUrl; + private SourceLocationType sourceLocationType; + private String zipFilePath; + + /** + * Must be specified if sourceLocationType is {@link SourceLocationType#REMOTE_REPOSITORY} + */ + private RemoteRepositoryInfo remoteRepositoryInfo; +} diff --git a/src/main/java/com/cx/restclient/ast/dto/common/Credentials.java b/src/main/java/com/cx/restclient/ast/dto/common/Credentials.java new file mode 100644 index 00000000..776e4bf9 --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/common/Credentials.java @@ -0,0 +1,8 @@ +package com.cx.restclient.ast.dto.common; + +public class Credentials { + + public String type; + public String value; + +} diff --git a/src/main/java/com/cx/restclient/ast/dto/common/GitCredentials.java b/src/main/java/com/cx/restclient/ast/dto/common/GitCredentials.java new file mode 100644 index 00000000..b1d3a003 --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/common/GitCredentials.java @@ -0,0 +1,11 @@ +package com.cx.restclient.ast.dto.common; + +import lombok.Builder; +import lombok.Getter; + +@Builder +@Getter +public class GitCredentials { + private String type; + private String value; +} diff --git a/src/main/java/com/cx/restclient/ast/dto/common/HandlerRef.java b/src/main/java/com/cx/restclient/ast/dto/common/HandlerRef.java new file mode 100644 index 00000000..c33675c2 --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/common/HandlerRef.java @@ -0,0 +1,11 @@ +package com.cx.restclient.ast.dto.common; + +import lombok.Builder; +import lombok.Getter; + +@Builder +@Getter +public class HandlerRef { + private String type; + private String value; +} diff --git a/src/main/java/com/cx/restclient/ast/dto/common/ProjectToScan.java b/src/main/java/com/cx/restclient/ast/dto/common/ProjectToScan.java new file mode 100644 index 00000000..547e576e --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/common/ProjectToScan.java @@ -0,0 +1,12 @@ +package com.cx.restclient.ast.dto.common; + +import lombok.Builder; +import lombok.Getter; + +@Builder +@Getter +public class ProjectToScan { + private String id; + private String type; + private ScanStartHandler handler; +} diff --git a/src/main/java/com/cx/restclient/ast/dto/common/RemoteRepositoryInfo.java b/src/main/java/com/cx/restclient/ast/dto/common/RemoteRepositoryInfo.java new file mode 100644 index 00000000..8b7ef4e2 --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/common/RemoteRepositoryInfo.java @@ -0,0 +1,29 @@ +package com.cx.restclient.ast.dto.common; + +import lombok.Getter; +import lombok.Setter; + +import java.io.Serializable; +import java.net.URL; + +/** + * Instructs AST scanners which repository should be scanned. + */ +@Getter +@Setter +public class RemoteRepositoryInfo implements Serializable { + /** + * A URL for which 'git clone' is possible. + */ + private URL url; + + private String branch; + + /** + * If access token is used instead of username/password, pass the token into this field. + * TODO: add a dedicated field for token. + */ + private String username; + + private String password; +} diff --git a/src/main/java/com/cx/restclient/ast/dto/common/ScanConfig.java b/src/main/java/com/cx/restclient/ast/dto/common/ScanConfig.java new file mode 100644 index 00000000..6995e4fa --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/common/ScanConfig.java @@ -0,0 +1,25 @@ +package com.cx.restclient.ast.dto.common; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +/** + * Defines a single scan engine that will be used during scan. + */ +@Builder +@Getter +@NoArgsConstructor +@AllArgsConstructor +public class ScanConfig { + /** + * Scan engine type. + */ + private String type; + + /** + * Engine-specific config. + */ + private ScanConfigValue value; +} diff --git a/src/main/java/com/cx/restclient/ast/dto/common/ScanConfigValue.java b/src/main/java/com/cx/restclient/ast/dto/common/ScanConfigValue.java new file mode 100644 index 00000000..88a0a543 --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/common/ScanConfigValue.java @@ -0,0 +1,8 @@ +package com.cx.restclient.ast.dto.common; + +/** + * Marker interface for scanner-specific scan configurations. + */ +public interface ScanConfigValue { + +} diff --git a/src/main/java/com/cx/restclient/ast/dto/common/ScanInfoResponse.java b/src/main/java/com/cx/restclient/ast/dto/common/ScanInfoResponse.java new file mode 100644 index 00000000..74a18f4a --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/common/ScanInfoResponse.java @@ -0,0 +1,9 @@ +package com.cx.restclient.ast.dto.common; + +import lombok.Getter; + +@Getter +public class ScanInfoResponse { + private String id; + private String status; +} diff --git a/src/main/java/com/cx/restclient/ast/dto/common/ScanStartHandler.java b/src/main/java/com/cx/restclient/ast/dto/common/ScanStartHandler.java new file mode 100644 index 00000000..2f282e2c --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/common/ScanStartHandler.java @@ -0,0 +1,23 @@ +package com.cx.restclient.ast.dto.common; + +import lombok.Builder; +import lombok.Getter; + +@Builder +@Getter +public class ScanStartHandler { + /** + * For local directory scan - the URL where the zipped directory has been uploaded. + * For remote repo scan - a URL for which 'git clone' is possible. + */ + private String url; + + /** + * For remote repo scan, contains a reference to a specific commit. + */ + private HandlerRef ref; + + private String username; + + private GitCredentials credentials; +} diff --git a/src/main/java/com/cx/restclient/ast/dto/common/ScanStatus.java b/src/main/java/com/cx/restclient/ast/dto/common/ScanStatus.java new file mode 100644 index 00000000..99b7544b --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/common/ScanStatus.java @@ -0,0 +1,10 @@ +package com.cx.restclient.ast.dto.common; + +public enum ScanStatus { + // Some of the statuses are not used in code, but they help to prevent the "unknown status" warnings. + CANCELED, + QUEUED, + RUNNING, + COMPLETED, + FAILED +} diff --git a/src/main/java/com/cx/restclient/ast/dto/common/StartScanRequest.java b/src/main/java/com/cx/restclient/ast/dto/common/StartScanRequest.java new file mode 100644 index 00000000..e2b6909e --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/common/StartScanRequest.java @@ -0,0 +1,20 @@ +package com.cx.restclient.ast.dto.common; + +import lombok.Builder; +import lombok.Getter; + +import java.util.List; + +@Builder +@Getter +public class StartScanRequest { + /** + * What to scan. + */ + private ProjectToScan project; + + /** + * How to scan. + */ + private List config; +} diff --git a/src/main/java/com/cx/restclient/ast/dto/common/SummaryResults.java b/src/main/java/com/cx/restclient/ast/dto/common/SummaryResults.java new file mode 100644 index 00000000..8fc296e4 --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/common/SummaryResults.java @@ -0,0 +1,12 @@ +package com.cx.restclient.ast.dto.common; + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class SummaryResults { + private int highVulnerabilityCount = 0; + private int mediumVulnerabilityCount = 0; + private int lowVulnerabilityCount = 0; +} diff --git a/src/main/java/com/cx/restclient/ast/dto/sast/AstSastConfig.java b/src/main/java/com/cx/restclient/ast/dto/sast/AstSastConfig.java new file mode 100644 index 00000000..d018ffe2 --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/sast/AstSastConfig.java @@ -0,0 +1,25 @@ +package com.cx.restclient.ast.dto.sast; + +import com.cx.restclient.ast.dto.common.ASTConfig; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.experimental.SuperBuilder; + +import java.io.Serializable; + +@SuperBuilder +@Getter +@Setter +@NoArgsConstructor +public class AstSastConfig extends ASTConfig implements Serializable { + private String clientSecret; + private String clientId; + private String presetName; + private boolean incremental; + + /** + * Used as a paging parameter in scan result requests. + */ + private int resultsPageSize; +} diff --git a/src/main/java/com/cx/restclient/ast/dto/sast/AstSastResults.java b/src/main/java/com/cx/restclient/ast/dto/sast/AstSastResults.java new file mode 100644 index 00000000..eed37969 --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/sast/AstSastResults.java @@ -0,0 +1,19 @@ +package com.cx.restclient.ast.dto.sast; + +import com.cx.restclient.ast.dto.sast.report.AstSastSummaryResults; +import com.cx.restclient.ast.dto.sast.report.Finding; +import com.cx.restclient.dto.Results; +import lombok.Getter; +import lombok.Setter; + +import java.io.Serializable; +import java.util.List; + +@Getter +@Setter +public class AstSastResults extends Results implements Serializable { + private String scanId; + private AstSastSummaryResults summary; + private String webReportLink; + private List findings; +} diff --git a/src/main/java/com/cx/restclient/ast/dto/sast/SastScanConfigValue.java b/src/main/java/com/cx/restclient/ast/dto/sast/SastScanConfigValue.java new file mode 100644 index 00000000..3a75535c --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/sast/SastScanConfigValue.java @@ -0,0 +1,23 @@ +package com.cx.restclient.ast.dto.sast; + +import com.cx.restclient.ast.dto.common.ScanConfigValue; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +/** + * AST-SAST-specific config parameters. Should be expanded with other supported properties. + */ +@Builder +@Getter +@NoArgsConstructor +@AllArgsConstructor +public class SastScanConfigValue implements ScanConfigValue { + private String presetName; + + /** + * Must be a string ("true" or "false"). + */ + private String incremental; +} diff --git a/src/main/java/com/cx/restclient/ast/dto/sast/report/AstSastSummaryResults.java b/src/main/java/com/cx/restclient/ast/dto/sast/report/AstSastSummaryResults.java new file mode 100644 index 00000000..13122e2d --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/sast/report/AstSastSummaryResults.java @@ -0,0 +1,18 @@ +package com.cx.restclient.ast.dto.sast.report; + +import com.cx.restclient.ast.dto.common.SummaryResults; +import lombok.Getter; +import lombok.Setter; + +import java.io.Serializable; +import java.util.List; + +/** + * Summary for external use as a part of general scan results. + */ +@Getter +@Setter +public class AstSastSummaryResults extends SummaryResults implements Serializable { + private List statusCounters; + private int totalCounter; +} diff --git a/src/main/java/com/cx/restclient/ast/dto/sast/report/Finding.java b/src/main/java/com/cx/restclient/ast/dto/sast/report/Finding.java new file mode 100644 index 00000000..e5212fd1 --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/sast/report/Finding.java @@ -0,0 +1,27 @@ +package com.cx.restclient.ast.dto.sast.report; + +import lombok.Getter; +import lombok.Setter; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +@Getter +@Setter +public class Finding implements Serializable { + private String queryID; + private String queryName; + private String severity; + private int cweID; + private int similarityID; + private int uniqueID; + private List nodes = new ArrayList<>(); + private String pathSystemID; + private String firstScanID; + private String firstFoundAt; + private String foundAt; + private String status; + private String description; + private String state; +} diff --git a/src/main/java/com/cx/restclient/ast/dto/sast/report/FindingNode.java b/src/main/java/com/cx/restclient/ast/dto/sast/report/FindingNode.java new file mode 100644 index 00000000..890092fa --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/sast/report/FindingNode.java @@ -0,0 +1,18 @@ +package com.cx.restclient.ast.dto.sast.report; +import lombok.Getter; +import lombok.Setter; + +import java.io.Serializable; + +@Getter +@Setter +public class FindingNode implements Serializable { + private int column; + private String fileName; + private String fullName; + private int length; + private int line; + private int methodLine; + private String name; + private String nodeSystemID; +} \ No newline at end of file diff --git a/src/main/java/com/cx/restclient/ast/dto/sast/report/QueryDescription.java b/src/main/java/com/cx/restclient/ast/dto/sast/report/QueryDescription.java new file mode 100644 index 00000000..257d613e --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/sast/report/QueryDescription.java @@ -0,0 +1,18 @@ +package com.cx.restclient.ast.dto.sast.report; + +import lombok.Getter; +import lombok.Setter; + +import java.io.Serializable; + +@Getter +@Setter +public class QueryDescription implements Serializable { + + private String queryId; + private String queryDescriptionId; + private String resultDescription; + private String risk; + private String cause; + private String generalRecommendations; +} diff --git a/src/main/java/com/cx/restclient/ast/dto/sast/report/ScanResultsResponse.java b/src/main/java/com/cx/restclient/ast/dto/sast/report/ScanResultsResponse.java new file mode 100644 index 00000000..d6b895ba --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/sast/report/ScanResultsResponse.java @@ -0,0 +1,15 @@ +package com.cx.restclient.ast.dto.sast.report; + +import lombok.Getter; +import lombok.Setter; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +@Getter +@Setter +public class ScanResultsResponse implements Serializable { + private List results = new ArrayList<>(); + private int totalCount; +} diff --git a/src/main/java/com/cx/restclient/ast/dto/sast/report/SeverityCounter.java b/src/main/java/com/cx/restclient/ast/dto/sast/report/SeverityCounter.java new file mode 100644 index 00000000..dac426ea --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/sast/report/SeverityCounter.java @@ -0,0 +1,13 @@ +package com.cx.restclient.ast.dto.sast.report; + +import lombok.Getter; +import lombok.Setter; + +import java.io.Serializable; + +@Getter +@Setter +public class SeverityCounter implements Serializable { + private String severity; + private int counter; +} diff --git a/src/main/java/com/cx/restclient/ast/dto/sast/report/SingleScanSummary.java b/src/main/java/com/cx/restclient/ast/dto/sast/report/SingleScanSummary.java new file mode 100644 index 00000000..c5bfa82e --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/sast/report/SingleScanSummary.java @@ -0,0 +1,18 @@ +package com.cx.restclient.ast.dto.sast.report; + +import lombok.Getter; +import lombok.Setter; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +@Getter +@Setter +public +class SingleScanSummary implements Serializable { + private String scanId; + private List severityCounters = new ArrayList<>(); + private List statusCounters = new ArrayList<>(); + private int totalCounter; +} diff --git a/src/main/java/com/cx/restclient/ast/dto/sast/report/StatusCounter.java b/src/main/java/com/cx/restclient/ast/dto/sast/report/StatusCounter.java new file mode 100644 index 00000000..ea5bd8e2 --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/sast/report/StatusCounter.java @@ -0,0 +1,13 @@ +package com.cx.restclient.ast.dto.sast.report; + +import lombok.Getter; +import lombok.Setter; + +import java.io.Serializable; + +@Getter +@Setter +public class StatusCounter implements Serializable { + private String status; + private int counter; +} diff --git a/src/main/java/com/cx/restclient/ast/dto/sast/report/SummaryResponse.java b/src/main/java/com/cx/restclient/ast/dto/sast/report/SummaryResponse.java new file mode 100644 index 00000000..c042cdb6 --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/sast/report/SummaryResponse.java @@ -0,0 +1,15 @@ +package com.cx.restclient.ast.dto.sast.report; + +import lombok.Getter; +import lombok.Setter; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +@Getter +@Setter +public class SummaryResponse implements Serializable { + private List scansSummaries = new ArrayList<>(); + private int totalCount; +} diff --git a/src/main/java/com/cx/restclient/ast/dto/sca/AstScaConfig.java b/src/main/java/com/cx/restclient/ast/dto/sca/AstScaConfig.java new file mode 100644 index 00000000..1a32afc3 --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/sca/AstScaConfig.java @@ -0,0 +1,42 @@ +package com.cx.restclient.ast.dto.sca; + +import java.io.Serializable; +import java.util.List; +import java.util.Map; + +import com.cx.restclient.ast.dto.common.ASTConfig; + +import lombok.Getter; +import lombok.Setter; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +@Getter +@Setter +@JsonIgnoreProperties(ignoreUnknown = true) +public class AstScaConfig extends ASTConfig implements Serializable { + + private String accessControlUrl; + private String username; + private String password; + private String tenant; + + /** + * true: upload all sources for scan + *
+ * false: only upload manifest and fingerprints for scan. Useful for customers that don't want their proprietary + * code to be uploaded into the cloud. + */ + private boolean includeSources; + + private String fingerprintsIncludePattern; + private String manifestsIncludePattern; + private String fingerprintFilePath; + private String sastProjectId; + private String sastProjectName; + private String sastServerUrl; + private String sastUsername; + private String sastPassword; + private Map envVariables; + private List configFilePaths; + +} diff --git a/src/main/java/com/cx/restclient/ast/dto/sca/AstScaResults.java b/src/main/java/com/cx/restclient/ast/dto/sca/AstScaResults.java new file mode 100644 index 00000000..aea17453 --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/sca/AstScaResults.java @@ -0,0 +1,49 @@ +package com.cx.restclient.ast.dto.sca; + +import com.cx.restclient.ast.dto.sca.report.AstScaSummaryResults; +import com.cx.restclient.ast.dto.sca.report.Finding; +import com.cx.restclient.ast.dto.sca.report.Package; +import com.cx.restclient.ast.dto.sca.report.PolicyEvaluation; +import com.cx.restclient.dto.Results; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.io.Serializable; +import java.util.List; + +@Builder +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +public class AstScaResults extends Results implements Serializable { + private String scanId; + private String reportId; + private AstScaSummaryResults summary; + private String webReportLink; + private List findings; + private List packages; + private boolean scaResultReady; + private int nonVulnerableLibraries; + private int vulnerableAndOutdated; + private List policyEvaluations; + private boolean policyViolated; + private boolean breakTheBuild; + + public void calculateVulnerableAndOutdatedPackages() { + int sum; + if (this.packages != null) { + for (Package pckg : this.packages) { + sum = pckg.getHighVulnerabilityCount() + pckg.getMediumVulnerabilityCount() + pckg.getLowVulnerabilityCount(); + if (sum == 0) { + this.nonVulnerableLibraries++; + } else if (sum > 0 && pckg.isOutdated()) { + this.vulnerableAndOutdated++; + } + } + } + } +} diff --git a/src/main/java/com/cx/restclient/ast/dto/sca/CreateProjectRequest.java b/src/main/java/com/cx/restclient/ast/dto/sca/CreateProjectRequest.java new file mode 100644 index 00000000..ad130f14 --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/sca/CreateProjectRequest.java @@ -0,0 +1,27 @@ +package com.cx.restclient.ast.dto.sca; + +import java.util.ArrayList; +import java.util.List; + +public class CreateProjectRequest { + private String name; + private List assignedTeams = new ArrayList<>(); + + public List getAssignedTeams() { + return assignedTeams; + } + + public void addAssignedTeams(String assignedTeam) { + this.assignedTeams.add(assignedTeam); + } + + public void setName(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + +} diff --git a/src/main/java/com/cx/restclient/ast/dto/sca/Project.java b/src/main/java/com/cx/restclient/ast/dto/sca/Project.java new file mode 100644 index 00000000..7fafedea --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/sca/Project.java @@ -0,0 +1,11 @@ +package com.cx.restclient.ast.dto.sca; + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class Project { + private String name; + private String id; +} diff --git a/src/main/java/com/cx/restclient/ast/dto/sca/ScaScanConfigValue.java b/src/main/java/com/cx/restclient/ast/dto/sca/ScaScanConfigValue.java new file mode 100644 index 00000000..83b53de4 --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/sca/ScaScanConfigValue.java @@ -0,0 +1,27 @@ +package com.cx.restclient.ast.dto.sca; + +import com.cx.restclient.ast.dto.common.ScanConfigValue; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; + +import java.util.Map; + +/** + * AST-SCA-specific config parameters. Should be expanded with other supported properties. + */ +@Builder +@Getter +@NoArgsConstructor +@AllArgsConstructor +public class ScaScanConfigValue implements ScanConfigValue { + + private String environmentVariables; + private String sastProjectId; + private String sastServerUrl; + private String sastUsername; + private String sastPassword; + private String sastProjectName; + +} diff --git a/src/main/java/com/cx/restclient/ast/dto/sca/report/AstScaSummaryResults.java b/src/main/java/com/cx/restclient/ast/dto/sca/report/AstScaSummaryResults.java new file mode 100644 index 00000000..a8435cca --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/sca/report/AstScaSummaryResults.java @@ -0,0 +1,99 @@ +package com.cx.restclient.ast.dto.sca.report; + + +import java.io.Serializable; + +public class AstScaSummaryResults implements Serializable { + private int totalPackages; + private int directPackages; + private String createdOn; + private double riskScore; + private int totalOutdatedPackages; + private int highVulnerabilityCount = 0; + private int mediumVulnerabilityCount = 0; + private int lowVulnerabilityCount = 0; + + public AstScaSummaryResults() { + } + + public AstScaSummaryResults(int totalPackages, int directPackages, String createdOn, double riskScore, int totalOutdatedPackages, int highVulnerabilityCount, int mediumVulnerabilityCount, int lowVulnerabilityCount) { + this.totalPackages = totalPackages; + this.directPackages = directPackages; + this.createdOn = createdOn; + this.riskScore = riskScore; + this.totalOutdatedPackages = totalOutdatedPackages; + this.highVulnerabilityCount = highVulnerabilityCount; + this.mediumVulnerabilityCount = mediumVulnerabilityCount; + this.lowVulnerabilityCount = lowVulnerabilityCount; + } + + public int getTotalOkLibraries() { + int totalOk = (totalPackages - (highVulnerabilityCount + mediumVulnerabilityCount + lowVulnerabilityCount)); + totalOk = Math.max(totalOk, 0); + return totalOk; + } + + public int getTotalPackages() { + return totalPackages; + } + + public void setTotalPackages(int totalPackages) { + this.totalPackages = totalPackages; + } + + public int getDirectPackages() { + return directPackages; + } + + public void setDirectPackages(int directPackages) { + this.directPackages = directPackages; + } + + public String getCreatedOn() { + return createdOn; + } + + public void setCreatedOn(String createdOn) { + this.createdOn = createdOn; + } + + public double getRiskScore() { + return riskScore; + } + + public void setRiskScore(double riskScore) { + this.riskScore = riskScore; + } + + public int getTotalOutdatedPackages() { + return totalOutdatedPackages; + } + + public void setTotalOutdatedPackages(int totalOutdatedPackages) { + this.totalOutdatedPackages = totalOutdatedPackages; + } + + public int getHighVulnerabilityCount() { + return highVulnerabilityCount; + } + + public void setHighVulnerabilityCount(int highVulnerabilityCount) { + this.highVulnerabilityCount = highVulnerabilityCount; + } + + public int getMediumVulnerabilityCount() { + return mediumVulnerabilityCount; + } + + public void setMediumVulnerabilityCount(int mediumVulnerabilityCount) { + this.mediumVulnerabilityCount = mediumVulnerabilityCount; + } + + public int getLowVulnerabilityCount() { + return lowVulnerabilityCount; + } + + public void setLowVulnerabilityCount(int lowVulnerabilityCount) { + this.lowVulnerabilityCount = lowVulnerabilityCount; + } +} diff --git a/src/main/java/com/cx/restclient/ast/dto/sca/report/DependencyPath.java b/src/main/java/com/cx/restclient/ast/dto/sca/report/DependencyPath.java new file mode 100644 index 00000000..fe30a269 --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/sca/report/DependencyPath.java @@ -0,0 +1,15 @@ +package com.cx.restclient.ast.dto.sca.report; + +import lombok.Getter; +import lombok.Setter; + +import java.io.Serializable; +import java.util.ArrayList; + +/** + * Added for readability. + */ +@Getter +@Setter +public class DependencyPath extends ArrayList implements Serializable { +} diff --git a/src/main/java/com/cx/restclient/ast/dto/sca/report/DependencyPathSegment.java b/src/main/java/com/cx/restclient/ast/dto/sca/report/DependencyPathSegment.java new file mode 100644 index 00000000..edbe02ef --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/sca/report/DependencyPathSegment.java @@ -0,0 +1,16 @@ +package com.cx.restclient.ast.dto.sca.report; + +import lombok.Getter; +import lombok.Setter; + +import java.io.Serializable; + +@Getter +@Setter +public class DependencyPathSegment implements Serializable { + private String id; + private String name; + private String version; + private boolean isResolved; + private boolean isDevelopment; +} diff --git a/src/main/java/com/cx/restclient/ast/dto/sca/report/Finding.java b/src/main/java/com/cx/restclient/ast/dto/sca/report/Finding.java new file mode 100644 index 00000000..4e2dad9d --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/sca/report/Finding.java @@ -0,0 +1,30 @@ +package com.cx.restclient.ast.dto.sca.report; + +import com.cx.restclient.dto.scansummary.Severity; +import lombok.Getter; +import lombok.Setter; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +/** + * This entity is called vulnerability in SCA API, but here it is called Finding for consistency. + * Indicates a specific type of vulnerability detected in a specific package. + */ +@Getter +@Setter +public class Finding implements Serializable { + private String id; + private String cveName; + private double score; + private Severity severity; + private String publishDate; + private List references = new ArrayList<>(); + private String description; + private String recommendations; + private String packageId; + private String similarityId; + private String fixResolutionText; + private boolean isIgnored; +} diff --git a/src/main/java/com/cx/restclient/ast/dto/sca/report/Package.java b/src/main/java/com/cx/restclient/ast/dto/sca/report/Package.java new file mode 100644 index 00000000..beafe48a --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/sca/report/Package.java @@ -0,0 +1,44 @@ +package com.cx.restclient.ast.dto.sca.report; + +import lombok.Getter; +import lombok.Setter; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +/** + * Info about a package that SCA retrieves by analyzing project dependencies. + */ +@Getter +@Setter +public class Package implements Serializable { + private String id; + private String name; + private String version; + private List licenses = new ArrayList<>(); + + /** + * The current values are [Filename, Sha1]. Not considered an enum in SCA API. + */ + private String matchType; + + private int highVulnerabilityCount; + private int mediumVulnerabilityCount; + private int lowVulnerabilityCount; + private int ignoredVulnerabilityCount; + private int numberOfVersionsSinceLastUpdate; + private String newestVersionReleaseDate; + private String newestVersion; + private boolean outdated; + private String releaseDate; + private String confidenceLevel; + private double riskScore; + private PackageSeverity severity; + private List locations = new ArrayList<>(); + private List dependencyPaths = new ArrayList<>(); + private String packageRepository; + private boolean isDirectDependency; + private boolean isDevelopment; + private PackageUsage packageUsage; +} diff --git a/src/main/java/com/cx/restclient/ast/dto/sca/report/PackageSeverity.java b/src/main/java/com/cx/restclient/ast/dto/sca/report/PackageSeverity.java new file mode 100644 index 00000000..cc6410f8 --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/sca/report/PackageSeverity.java @@ -0,0 +1,12 @@ +package com.cx.restclient.ast.dto.sca.report; + +public enum PackageSeverity { + /** + * Package was scanned but no vulnerabilities were detected. + */ + NONE, + + LOW, + MEDIUM, + HIGH +} diff --git a/src/main/java/com/cx/restclient/ast/dto/sca/report/PackageUsage.java b/src/main/java/com/cx/restclient/ast/dto/sca/report/PackageUsage.java new file mode 100644 index 00000000..4a9c90f6 --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/sca/report/PackageUsage.java @@ -0,0 +1,13 @@ +package com.cx.restclient.ast.dto.sca.report; + +import lombok.Getter; +import lombok.Setter; + +import java.io.Serializable; + +@Getter +@Setter +public class PackageUsage implements Serializable { + private String usageType; + private String packageId; +} diff --git a/src/main/java/com/cx/restclient/ast/dto/sca/report/PolicyAction.java b/src/main/java/com/cx/restclient/ast/dto/sca/report/PolicyAction.java new file mode 100644 index 00000000..16c6f5c0 --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/sca/report/PolicyAction.java @@ -0,0 +1,18 @@ +package com.cx.restclient.ast.dto.sca.report; + +import java.io.Serializable; + + +public class PolicyAction implements Serializable { + private boolean breakBuild; + + public final boolean isBreakBuild() { + return breakBuild; + } + + public final void setBreakBuild(boolean breakBuild) { + this.breakBuild = breakBuild; + } + + +} diff --git a/src/main/java/com/cx/restclient/ast/dto/sca/report/PolicyEvaluation.java b/src/main/java/com/cx/restclient/ast/dto/sca/report/PolicyEvaluation.java new file mode 100644 index 00000000..46e1a8ad --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/sca/report/PolicyEvaluation.java @@ -0,0 +1,58 @@ +package com.cx.restclient.ast.dto.sca.report; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +//cannot use lombok because the way boolean members in the APi payload are named +//Jackson ObjectMapper looks for setter/getter with literally same. + +public class PolicyEvaluation implements Serializable { + private String id; + private String description; + private String name; + private boolean isViolated; + private PolicyAction actions; + private List rules = new ArrayList<>(); + + public final String getId() { + return id; + } + public final void setId(String id) { + this.id = id; + } + public final String getDescription() { + return description; + } + public final void setDescription(String description) { + this.description = description; + } + public final String getName() { + return name; + } + public final void setName(String name) { + this.name = name; + } + public final boolean getIsViolated() { + return isViolated; + } + + public final void setIsViolated(boolean isViolated) { + this.isViolated = isViolated; + } + + public final PolicyAction getActions() { + return actions; + } + public final void setActions(PolicyAction actions) { + this.actions = actions; + } + public final List getRules() { + return rules; + } + public final void setRules(List rules) { + this.rules = rules; + } + + +} diff --git a/src/main/java/com/cx/restclient/ast/dto/sca/report/PolicyRule.java b/src/main/java/com/cx/restclient/ast/dto/sca/report/PolicyRule.java new file mode 100644 index 00000000..cf768d24 --- /dev/null +++ b/src/main/java/com/cx/restclient/ast/dto/sca/report/PolicyRule.java @@ -0,0 +1,30 @@ +package com.cx.restclient.ast.dto.sca.report; + +import java.io.Serializable; + +public class PolicyRule implements Serializable { + private String id; + private boolean isViolated; + private String name; + + public final String getId() { + return id; + } + public final void setId(String id) { + this.id = id; + } + public final boolean getIsViolated() { + return isViolated; + } + public final void setIsViolated(boolean isViolated) { + this.isViolated = isViolated; + } + public final String getName() { + return name; + } + public final void setName(String name) { + this.name = name; + } + + +} diff --git a/src/main/java/com/cx/restclient/common/CxPARAM.java b/src/main/java/com/cx/restclient/common/CxPARAM.java index 7536a0f8..b692c562 100644 --- a/src/main/java/com/cx/restclient/common/CxPARAM.java +++ b/src/main/java/com/cx/restclient/common/CxPARAM.java @@ -6,14 +6,32 @@ * Created by Galn on 14/02/2018. */ public abstract class CxPARAM { - public static final String AUTHENTICATION = "auth/identity/connect/token"; - public static final String ORIGIN_HEADER = "cxOrigin"; + public static final String AUTHENTICATION = "identity/connect/token"; + public static final String REVOCATION = "auth/identity/connect/revocation"; + public static final String SSO_AUTHENTICATION = "auth/identity/externalLogin"; public static final String CXPRESETS = "sast/presets"; public static final String CXTEAMS = "auth/teams"; public static final String CREATE_PROJECT = "projects";//Create new project (default preset and configuration) + public static final String CX_VERSION = "system/version"; + + public static final String CX_ARM_URL = "/Configurations/Portal"; + public static final String CX_ARM_VIOLATION = "/cxarm/policymanager/projects/{projectId}/violations?provider={provider}"; + + public static final String BROWSER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"; + + public static final String CX_REPORT_LOCATION = File.separator + "Checkmarx" + File.separator + "Reports"; - public static final String CX_ARM_URL ="/Configurations/Portal"; - public static final String CX_ARM_VIOLATION ="/cxarm/policymanager/projects/{projectId}/violations?provider={provider}"; + public static final String ORIGIN_HEADER = "cxOrigin"; + public static final String ORIGIN_URL_HEADER = "cxOriginUrl"; + public static final String CSRF_TOKEN_HEADER = "CXCSRFToken"; + public static final String PROJECT_POLICY_VIOLATED_STATUS = "Project policy status : violated"; + public static final String PROJECT_POLICY_COMPLIANT_STATUS = "Project policy status : compliant"; + + public static final String DENY_NEW_PROJECT_ERROR = "Creation of the new project [{projectName}] is not authorized. " + + "Please use an existing project. \nYou can enable the creation of new projects by disabling" + "" + + " the Deny new Checkmarx projects creation checkbox in the Checkmarx plugin global settings.\n"; + public static final String TEAM_PATH = "cxTeamPath"; + } diff --git a/src/main/java/com/cx/restclient/common/ErrorUtil.java b/src/main/java/com/cx/restclient/common/ErrorUtil.java deleted file mode 100644 index 208b69e8..00000000 --- a/src/main/java/com/cx/restclient/common/ErrorUtil.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.cx.restclient.common; - -import org.apache.http.HttpStatus; - -import java.util.HashSet; -import java.util.Set; - -/** - * Created by shaulv on 6/20/2018. - */ -public class ErrorUtil { - - private ErrorUtil() { - - } - - private static Set serverErrorCodes = new HashSet<>(); - - static { - serverErrorCodes.add(HttpStatus.SC_INTERNAL_SERVER_ERROR); - serverErrorCodes.add(HttpStatus.SC_NOT_IMPLEMENTED); - serverErrorCodes.add(HttpStatus.SC_BAD_GATEWAY); - serverErrorCodes.add(HttpStatus.SC_SERVICE_UNAVAILABLE); - serverErrorCodes.add(HttpStatus.SC_GATEWAY_TIMEOUT); - serverErrorCodes.add(HttpStatus.SC_HTTP_VERSION_NOT_SUPPORTED); - serverErrorCodes.add(HttpStatus.SC_INSUFFICIENT_STORAGE); - } - - public static boolean isServerErrorCodes(int errorCodes) { - if(serverErrorCodes.contains(errorCodes)) { - return true; - } - return false; - } -} diff --git a/src/main/java/com/cx/restclient/common/Scanner.java b/src/main/java/com/cx/restclient/common/Scanner.java new file mode 100644 index 00000000..4515b06f --- /dev/null +++ b/src/main/java/com/cx/restclient/common/Scanner.java @@ -0,0 +1,23 @@ +package com.cx.restclient.common; + +import com.cx.restclient.dto.Results; +import com.cx.restclient.sast.utils.State; + +/** + * Common functionality for vulnerability scanners. + */ +public interface Scanner { + Results init(); + + Results initiateScan(); + + Results waitForScanResults(); + + Results getLatestScanResults(); + + void close(); + + default State getState() { + return State.SUCCESS; + } +} diff --git a/src/main/java/com/cx/restclient/common/ShragaUtils.java b/src/main/java/com/cx/restclient/common/ShragaUtils.java index a3a37e9c..65b9af30 100644 --- a/src/main/java/com/cx/restclient/common/ShragaUtils.java +++ b/src/main/java/com/cx/restclient/common/ShragaUtils.java @@ -1,8 +1,5 @@ package com.cx.restclient.common; -import com.cx.restclient.configuration.CxScanConfig; -import com.cx.restclient.osa.dto.OSAResults; -import com.cx.restclient.sast.dto.SASTResults; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -17,66 +14,11 @@ * Date: 4/12/2018. */ public abstract class ShragaUtils { - //Util methods - public static boolean isThresholdExceeded(CxScanConfig config, SASTResults sastResults, OSAResults osaResults, StringBuilder res) { - boolean thresholdExceeded = false; - if (config.isSASTThresholdEffectivelyEnabled() && sastResults != null && sastResults.isSastResultsReady()) { - thresholdExceeded = isSeverityExceeded(sastResults.getHigh(), config.getSastHighThreshold(), res, "high", "CxSAST "); - thresholdExceeded |= isSeverityExceeded(sastResults.getMedium(), config.getSastMediumThreshold(), res, "medium", "CxSAST "); - thresholdExceeded |= isSeverityExceeded(sastResults.getLow(), config.getSastLowThreshold(), res, "low", "CxSAST "); - } - if (config.isOSAThresholdEffectivelyEnabled() && osaResults != null && osaResults.isOsaResultsReady()) { - thresholdExceeded |= isSeverityExceeded(osaResults.getResults().getTotalHighVulnerabilities(), config.getOsaHighThreshold(), res, "high", "CxOSA "); - thresholdExceeded |= isSeverityExceeded(osaResults.getResults().getTotalMediumVulnerabilities(), config.getOsaMediumThreshold(), res, "medium", "CxOSA "); - thresholdExceeded |= isSeverityExceeded(osaResults.getResults().getTotalLowVulnerabilities(), config.getOsaLowThreshold(), res, "low", "CxOSA "); - } - return thresholdExceeded; - } - - public static boolean isThresholdForNewResultExceeded(CxScanConfig config, SASTResults sastResults, StringBuilder res) { - boolean exceeded = false; - - if (sastResults != null && sastResults.isSastResultsReady() && config.getSastNewResultsThresholdEnabled()) { - String severity = config.getSastNewResultsThresholdSeverity(); - - if ("LOW".equals(severity)) { - if (sastResults.getNewLow() > 0) { - res.append("One or more new results of low severity\n"); - exceeded = true; - } - severity = "MEDIUM"; - } - - if ("MEDIUM".equals(severity)) { - if (sastResults.getNewMedium() > 0) { - res.append("One or more new results of medium severity\n"); - exceeded = true; - } - severity = "HIGH"; - } - - if ("HIGH".equals(severity)) { - if (sastResults.getNewHigh() > 0) { - res.append("One or more New results of high severity\n"); - exceeded = true; - } - } - } - - return exceeded; - } - - private static boolean isSeverityExceeded(int result, Integer threshold, StringBuilder res, String severity, String severityType) { - boolean fail = false; - if (threshold != null && result > threshold) { - res.append(severityType).append(severity).append(" severity results are above threshold. Results: ").append(result).append(". Threshold: ").append(threshold).append("\n"); - fail = true; - } - return fail; - } - public static Map> generateIncludesExcludesPatternLists(String folderExclusions, String filterPattern, Logger log) { + folderExclusions = removeSpaceAndNewLine(folderExclusions); + filterPattern = removeSpaceAndNewLine(filterPattern); + String excludeFoldersPattern = processExcludeFolders(folderExclusions, log); String combinedPatterns = ""; @@ -93,6 +35,13 @@ public static Map> generateIncludesExcludesPatternLists(Str return convertPatternsToLists(combinedPatterns); } + public static String removeSpaceAndNewLine(String string){ + if(string!=null){ + string = string.replace("\\s","").replace("\n", "").replace("\r", "").replace(" ","").replace("\t",""); + } + return string; + } + public static String processExcludeFolders(String folderExclusions, Logger log) { if (StringUtils.isEmpty(folderExclusions)) { return ""; @@ -112,8 +61,10 @@ public static String processExcludeFolders(String folderExclusions, Logger log) log.info("Exclude folders converted to: '" + result.toString() + "'"); return result.toString(); } + public static final String INCLUDES_LIST = "includes"; public static final String EXCLUDES_LIST = "excludes"; + public static Map> convertPatternsToLists(String filterPatterns) { filterPatterns = StringUtils.defaultString(filterPatterns); List inclusions = new ArrayList(); @@ -122,10 +73,10 @@ public static Map> convertPatternsToLists(String filterPatt for (String filter : filters) { if (StringUtils.isNotEmpty(filter)) { if (!filter.startsWith("!")) { - inclusions.add(filter); + inclusions.add(filter.trim()); } else if (filter.length() > 1) { filter = filter.substring(1); // Trim the "!" - exclusions.add(filter); + exclusions.add(filter.trim()); } } } @@ -148,4 +99,15 @@ public static String formatDate(String date, String fromFormat, String toFormat) } return ret; } + + public static String getTimestampSince(long startTimeSec) { + long elapsedSec = System.currentTimeMillis() / 1000 - startTimeSec; + long hours = elapsedSec / 3600; + long minutes = elapsedSec % 3600 / 60; + long seconds = elapsedSec % 60; + String hoursStr = (hours < 10) ? ("0" + hours) : (Long.toString(hours)); + String minutesStr = (minutes < 10) ? ("0" + minutes) : (Long.toString(minutes)); + String secondsStr = (seconds < 10) ? ("0" + seconds) : (Long.toString(seconds)); + return String.format("%s:%s:%s", hoursStr, minutesStr, secondsStr); + } } diff --git a/src/main/java/com/cx/restclient/common/Waiter.java b/src/main/java/com/cx/restclient/common/Waiter.java index e852695f..7aada21d 100644 --- a/src/main/java/com/cx/restclient/common/Waiter.java +++ b/src/main/java/com/cx/restclient/common/Waiter.java @@ -4,6 +4,7 @@ import com.cx.restclient.dto.Status; import com.cx.restclient.exception.CxClientException; import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.Date; @@ -11,48 +12,56 @@ /** * Created by Galn on 13/02/2018. */ -public abstract class Waiter { +public abstract class Waiter { - private int retry = 5; + public static final Logger log = LoggerFactory.getLogger(Waiter.class); + + private static final String FAILED_MSG = "Failed to get status from "; + + private int retry; private String scanType; private int sleepIntervalSec; - public Waiter(String scanType, int interval) { + public Waiter(String scanType, int interval, int retry) { this.scanType = scanType; this.sleepIntervalSec = interval; + this.retry = retry; } private long startTimeSec; - protected Status status = null; - - public T waitForTaskToFinish(String taskId, Integer scanTimeoutSec, Logger log) throws CxClientException, InterruptedException { + public T waitForTaskToFinish(String taskId, Integer scanTimeoutSec, Logger log) throws CxClientException { startTimeSec = System.currentTimeMillis() / 1000; long elapsedTimeSec = 0L; - status = Status.IN_PROGRESS; - T obj = null; - - while (status.equals(Status.IN_PROGRESS) && (scanTimeoutSec <= 0 || elapsedTimeSec < scanTimeoutSec)) { - Thread.sleep(sleepIntervalSec * 1000); - try { - obj = getStatus(taskId); - status = ((BaseStatus) obj).getBaseStatus(); - } catch (Exception e) { - log.debug("Failed to get status from " + scanType + ". retrying (" + (retry - 1) + " tries left). Error message: " + e.getMessage()); - if (retry <= 0) { - throw new CxClientException("Failed to get status from " + scanType + ". Error message: " + e.getMessage(), e); + T statusResponse = null; + + try { + do { + try { + Thread.sleep((long) sleepIntervalSec * 1000); + statusResponse = getStatus(taskId); + } catch (Exception e) { + log.debug(FAILED_MSG + scanType + ". retrying (" + (retry - 1) + " tries left). Error message: " + e.getMessage()); + retry--; + if (retry <= 0) { + throw new CxClientException(FAILED_MSG + scanType + ". Error message: " + e.getMessage(), e); + } + if (statusResponse == null || (statusResponse.getBaseStatus() == null)) { + statusResponse = (T) new BaseStatus(Status.IN_PROGRESS); + } + continue; } - retry--; - continue; - } - elapsedTimeSec = (new Date()).getTime() / 1000 - startTimeSec; - printProgress(obj); + elapsedTimeSec = (new Date()).getTime() / 1000 - startTimeSec; + printProgress(statusResponse); + } while (isTaskInProgress(statusResponse) && (scanTimeoutSec <= 0 || elapsedTimeSec < scanTimeoutSec)); + if (scanTimeoutSec > 0 && scanTimeoutSec <= elapsedTimeSec) { + throw new CxClientException("Failed to perform " + scanType + ": " + scanType + " has been automatically aborted: reached the user-specified timeout (" + scanTimeoutSec / 60 + " minutes)"); + } + } catch (Exception e) { + throw new CxClientException(FAILED_MSG + scanType + ". Error message: " + e.getMessage(), e); } - if (scanTimeoutSec > 0 && scanTimeoutSec <= elapsedTimeSec) { - throw new CxClientException( "Failed to perform " + scanType + ": " + scanType + " has been automatically aborted: reached the user-specified timeout (" + scanTimeoutSec / 60 + " minutes)"); - } - return resolveStatus(obj); + return resolveStatus(statusResponse); } public abstract T getStatus(String id) throws CxClientException, IOException; @@ -61,15 +70,13 @@ public T waitForTaskToFinish(String taskId, Integer scanTimeoutSec, Logger log) public abstract T resolveStatus(T status) throws CxClientException; - public Status getStatus() { - return status; - } - - public void setStatus(Status status) { - this.status = status; + public boolean isTaskInProgress(T statusResponse) { + Status status = statusResponse.getBaseStatus(); + return status.equals(Status.IN_PROGRESS); } public long getStartTimeSec() { return startTimeSec; } + } diff --git a/src/main/java/com/cx/restclient/common/summary/DependencyScanResult.java b/src/main/java/com/cx/restclient/common/summary/DependencyScanResult.java new file mode 100644 index 00000000..bbafb917 --- /dev/null +++ b/src/main/java/com/cx/restclient/common/summary/DependencyScanResult.java @@ -0,0 +1,209 @@ +package com.cx.restclient.common.summary; + +import com.cx.restclient.dto.Results; +import com.cx.restclient.dto.ScannerType; +import com.cx.restclient.dto.scansummary.Severity; +import com.cx.restclient.osa.dto.CVEReportTableRow; +import com.cx.restclient.osa.dto.OSAResults; +import com.cx.restclient.ast.dto.sca.AstScaResults; +import com.cx.restclient.ast.dto.sca.report.Finding; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +import static com.cx.restclient.common.ShragaUtils.formatDate; + +public class DependencyScanResult extends Results implements Serializable { + private ScannerType scannerType; + private boolean resultReady; + private int highVulnerability; + private int mediumVulnerability; + private int lowVulnerability; + private String summaryLink; + private int vulnerableAndOutdated; + private int nonVulnerableLibraries; + private String scanStartTime; + private String scanEndTime; + private List dependencyHighCVEReportTable = new ArrayList<>(); + private List dependencyMediumCVEReportTable = new ArrayList<>(); + private List dependencyLowCVEReportTable = new ArrayList<>(); + private int totalLibraries; + + DependencyScanResult(){} + + DependencyScanResult(AstScaResults scaResults){ + scaResults.calculateVulnerableAndOutdatedPackages(); + this.scannerType = ScannerType.AST_SCA; + this.highVulnerability = scaResults.getSummary().getHighVulnerabilityCount(); + this.mediumVulnerability = scaResults.getSummary().getMediumVulnerabilityCount(); + this.lowVulnerability = scaResults.getSummary().getLowVulnerabilityCount(); + this.resultReady = scaResults.isScaResultReady(); + this.summaryLink = scaResults.getWebReportLink(); + this.vulnerableAndOutdated = scaResults.getVulnerableAndOutdated(); + this.nonVulnerableLibraries = scaResults.getNonVulnerableLibraries(); + this.scanStartTime = formatDate(scaResults.getSummary().getCreatedOn(), "yyyy-MM-dd'T'HH:mm:ss.SSSSSSS", "dd/MM/yy HH:mm"); + this.scanEndTime =""; + this.setDependencyCVEReportTableSCA(scaResults.getFindings()); + this.setTotalLibraries(scaResults.getSummary().getTotalPackages()); + } + + + DependencyScanResult(OSAResults osaResults){ + this.scannerType = ScannerType.OSA; + this.highVulnerability = osaResults.getResults().getTotalHighVulnerabilities(); + this.mediumVulnerability = osaResults.getResults().getTotalMediumVulnerabilities(); + this.lowVulnerability = osaResults.getResults().getTotalLowVulnerabilities(); + this.resultReady = osaResults.isOsaResultsReady(); + this.summaryLink = osaResults.getOsaProjectSummaryLink(); + this.vulnerableAndOutdated = osaResults.getResults().getVulnerableAndOutdated(); + this.nonVulnerableLibraries = osaResults.getResults().getNonVulnerableLibraries(); + this.scanStartTime =osaResults.getScanStartTime(); + this.scanEndTime = osaResults.getScanEndTime(); + this.setDependencyCVEReportTableOsa(osaResults.getOsaLowCVEReportTable(),osaResults.getOsaMediumCVEReportTable(),osaResults.getOsaHighCVEReportTable()); + this.setTotalLibraries(osaResults.getResults().getTotalLibraries()); + } + + public void setDependencyCVEReportTableOsa(List osaCVEResultsLow,List osaCVEResultsMedium,List osaCVEResultsHigh){ + CVEReportTableRow row; + for(CVEReportTableRow lowCVE :osaCVEResultsLow ){ + row = lowCVE; + this.dependencyLowCVEReportTable.add(row); + } + for(CVEReportTableRow mediumCVE :osaCVEResultsMedium ){ + row = mediumCVE; + this.dependencyMediumCVEReportTable.add(row); + } + for(CVEReportTableRow highCVE :osaCVEResultsHigh ){ + row = highCVE; + this.dependencyHighCVEReportTable.add(row); + } + } + + public void setDependencyCVEReportTableSCA(List scaFindings){ + CVEReportTableRow row; + for(Finding scaFinding :scaFindings ){ + row =new CVEReportTableRow(scaFinding); + if(scaFinding.getSeverity() == Severity.LOW){ + this.dependencyLowCVEReportTable.add(row); + }else if(scaFinding.getSeverity() == Severity.MEDIUM){ + this.dependencyMediumCVEReportTable.add(row); + }else if(scaFinding.getSeverity() == Severity.HIGH){ + this.dependencyHighCVEReportTable.add(row); + } + } + } + + public ScannerType getScannerType() { + return scannerType; + } + + public void setScannerType(ScannerType scannerType) { + this.scannerType = scannerType; + } + + public boolean isResultReady() { + return resultReady; + } + + public void setResultReady(boolean resultReady) { + this.resultReady = resultReady; + } + + public int getHighVulnerability() { + return highVulnerability; + } + + public void setHighVulnerability(int highVulnerability) { + this.highVulnerability = highVulnerability; + } + + public int getMediumVulnerability() { + return mediumVulnerability; + } + + public void setMediumVulnerability(int mediumVulnerability) { + this.mediumVulnerability = mediumVulnerability; + } + + public int getLowVulnerability() { + return lowVulnerability; + } + + public void setLowVulnerability(int lowVulnerability) { + this.lowVulnerability = lowVulnerability; + } + + public String getSummaryLink() { + return summaryLink; + } + + public void setSummaryLink(String summaryLink) { + this.summaryLink = summaryLink; + } + + public int getVulnerableAndOutdated() { + return vulnerableAndOutdated; + } + + public void setVulnerableAndOutdated(int vulnerableAndOutdated) { + this.vulnerableAndOutdated = vulnerableAndOutdated; + } + + public int getNonVulnerableLibraries() { + return nonVulnerableLibraries; + } + + public void setNonVulnerableLibraries(int nonVulnerableLibraries) { + this.nonVulnerableLibraries = nonVulnerableLibraries; + } + + public String getScanStartTime() { + return scanStartTime; + } + + public void setScanStartTime(String scanStartTime) { + this.scanStartTime = scanStartTime; + } + + public String getScanEndTime() { + return scanEndTime; + } + + public void setScanEndTime(String scanEndTime) { + this.scanEndTime = scanEndTime; + } + + public List getDependencyHighCVEReportTable() { + return dependencyHighCVEReportTable; + } + + public void setDependencyHighCVEReportTable(List dependencyHighCVEReportTable) { + this.dependencyHighCVEReportTable = dependencyHighCVEReportTable; + } + + public List getDependencyMediumCVEReportTable() { + return dependencyMediumCVEReportTable; + } + + public void setDependencyMediumCVEReportTable(List dependencyMediumCVEReportTable) { + this.dependencyMediumCVEReportTable = dependencyMediumCVEReportTable; + } + + public List getDependencyLowCVEReportTable() { + return dependencyLowCVEReportTable; + } + + public void setDependencyLowCVEReportTable(List dependencyLowCVEReportTable) { + this.dependencyLowCVEReportTable = dependencyLowCVEReportTable; + } + + public int getTotalLibraries() { + return totalLibraries; + } + + public void setTotalLibraries(int totalLibraries) { + this.totalLibraries = totalLibraries; + } + +} diff --git a/src/main/java/com/cx/restclient/common/summary/SummaryUtils.java b/src/main/java/com/cx/restclient/common/summary/SummaryUtils.java index e7fd16c2..96e64a56 100644 --- a/src/main/java/com/cx/restclient/common/summary/SummaryUtils.java +++ b/src/main/java/com/cx/restclient/common/summary/SummaryUtils.java @@ -1,10 +1,12 @@ package com.cx.restclient.common.summary; -import com.cx.restclient.common.ShragaUtils; import com.cx.restclient.configuration.CxScanConfig; +import com.cx.restclient.cxArm.dto.Policy; + +import com.cx.restclient.dto.scansummary.ScanSummary; import com.cx.restclient.osa.dto.OSAResults; -import com.cx.restclient.osa.dto.OSASummaryResults; import com.cx.restclient.sast.dto.SASTResults; +import com.cx.restclient.ast.dto.sca.AstScaResults; import freemarker.template.Configuration; import freemarker.template.Template; import freemarker.template.TemplateException; @@ -14,90 +16,188 @@ import java.io.StringWriter; import java.util.HashMap; import java.util.Map; +import java.util.stream.Collectors; public abstract class SummaryUtils { + private SummaryUtils() { + } - public static String generateSummary(SASTResults sastResults, OSAResults osaResults, CxScanConfig config) throws IOException, TemplateException { + public static String generateSummary(SASTResults sastResults, OSAResults osaResults, AstScaResults scaResults, CxScanConfig config) throws IOException, TemplateException { Configuration cfg = new Configuration(new Version("2.3.23")); - cfg.setClassForTemplateLoading(SummaryUtils.class, "/com/cx/report/"); + cfg.setClassForTemplateLoading(SummaryUtils.class, "/com/cx/report"); Template template = cfg.getTemplate("report.ftl"); - Map templateData = new HashMap(); + Map templateData = new HashMap<>(); templateData.put("config", config); - templateData.put("sast", sastResults); - templateData.put("osa", osaResults); + templateData.put("sast", sastResults != null ? sastResults : new SASTResults()); + + // TODO: null value for "osa" should be handled inside the template. + templateData.put("osa", osaResults != null ? osaResults : new OSAResults()); + templateData.put("sca", scaResults != null ? scaResults : new AstScaResults()); + + DependencyScanResult dependencyScanResult = resolveDependencyResult(osaResults, scaResults); + + templateData.put("dependencyResult", dependencyScanResult != null ? dependencyScanResult : new DependencyScanResult()); + + + ScanSummary scanSummary = new ScanSummary(config, sastResults, osaResults, scaResults); //calculated params: boolean buildFailed = false; boolean policyViolated = false; + int policyViolatedCount; //sast: - if (config.getSastEnabled() && sastResults.isSastResultsReady()) { - boolean sastThresholdExceeded = ShragaUtils.isThresholdExceeded(config, sastResults, null, new StringBuilder()); - boolean sastNewResultsExceeded = ShragaUtils.isThresholdForNewResultExceeded(config, sastResults, new StringBuilder()); - templateData.put("sastThresholdExceeded", sastThresholdExceeded); - templateData.put("sastNewResultsExceeded", sastNewResultsExceeded); - buildFailed = sastThresholdExceeded || sastNewResultsExceeded; - //calculate sast bars: - float maxCount = Math.max(sastResults.getHigh(), Math.max(sastResults.getMedium(), sastResults.getLow())); - float sastBarNorm = maxCount * 10f / 9f; - - //sast high bars - float sastHighTotalHeight = (float) sastResults.getHigh() / sastBarNorm * 238f; - float sastHighNewHeight = calculateNewBarHeight(sastResults.getNewHigh(), sastResults.getHigh(), sastHighTotalHeight); - float sastHighRecurrentHeight = sastHighTotalHeight - sastHighNewHeight; - templateData.put("sastHighTotalHeight", sastHighTotalHeight); - templateData.put("sastHighNewHeight", sastHighNewHeight); - templateData.put("sastHighRecurrentHeight", sastHighRecurrentHeight); - /*if (config.getEnablePolicyViolations() && !sastResults.getSastViolations().isEmpty()){ - policyViolated = true; - }*/ - - //sast medium bars - float sastMediumTotalHeight = (float) sastResults.getMedium() / sastBarNorm * 238f; - float sastMediumNewHeight = calculateNewBarHeight(sastResults.getNewMedium(), sastResults.getMedium(), sastMediumTotalHeight); - float sastMediumRecurrentHeight = sastMediumTotalHeight - sastMediumNewHeight; - templateData.put("sastMediumTotalHeight", sastMediumTotalHeight); - templateData.put("sastMediumNewHeight", sastMediumNewHeight); - templateData.put("sastMediumRecurrentHeight", sastMediumRecurrentHeight); - - //sast low bars - float sastLowTotalHeight = (float) sastResults.getLow() / sastBarNorm * 238f; - float sastLowNewHeight = calculateNewBarHeight(sastResults.getNewLow(), sastResults.getLow(), sastLowTotalHeight); - float sastLowRecurrentHeight = sastLowTotalHeight - sastLowNewHeight; - templateData.put("sastLowTotalHeight", sastLowTotalHeight); - templateData.put("sastLowNewHeight", sastLowNewHeight); - templateData.put("sastLowRecurrentHeight", sastLowRecurrentHeight); + if (config.isSastEnabled()) { + if (sastResults != null && sastResults.isSastResultsReady()) { + boolean sastThresholdExceeded = scanSummary.isSastThresholdExceeded(); + boolean sastNewResultsExceeded = scanSummary.isSastThresholdForNewResultsExceeded(); + templateData.put("sastThresholdExceeded", sastThresholdExceeded); + templateData.put("sastNewResultsExceeded", sastNewResultsExceeded); + buildFailed = sastThresholdExceeded || sastNewResultsExceeded; + //calculate sast bars: + float maxCount = Math.max(sastResults.getHigh(), Math.max(sastResults.getMedium(), sastResults.getLow())); + float sastBarNorm = maxCount * 10f / 9f; + + //sast high bars + float sastHighTotalHeight = (float) sastResults.getHigh() / sastBarNorm * 238f; + float sastHighNewHeight = calculateNewBarHeight(sastResults.getNewHigh(), sastResults.getHigh(), sastHighTotalHeight); + float sastHighRecurrentHeight = sastHighTotalHeight - sastHighNewHeight; + templateData.put("sastHighTotalHeight", sastHighTotalHeight); + templateData.put("sastHighNewHeight", sastHighNewHeight); + templateData.put("sastHighRecurrentHeight", sastHighRecurrentHeight); + + //sast medium bars + float sastMediumTotalHeight = (float) sastResults.getMedium() / sastBarNorm * 238f; + float sastMediumNewHeight = calculateNewBarHeight(sastResults.getNewMedium(), sastResults.getMedium(), sastMediumTotalHeight); + float sastMediumRecurrentHeight = sastMediumTotalHeight - sastMediumNewHeight; + templateData.put("sastMediumTotalHeight", sastMediumTotalHeight); + templateData.put("sastMediumNewHeight", sastMediumNewHeight); + templateData.put("sastMediumRecurrentHeight", sastMediumRecurrentHeight); + + //sast low bars + float sastLowTotalHeight = (float) sastResults.getLow() / sastBarNorm * 238f; + float sastLowNewHeight = calculateNewBarHeight(sastResults.getNewLow(), sastResults.getLow(), sastLowTotalHeight); + float sastLowRecurrentHeight = sastLowTotalHeight - sastLowNewHeight; + templateData.put("sastLowTotalHeight", sastLowTotalHeight); + templateData.put("sastLowNewHeight", sastLowNewHeight); + templateData.put("sastLowRecurrentHeight", sastLowRecurrentHeight); + } else { + buildFailed = true; + } } +/* //osa: - if (config.getOsaEnabled() && osaResults.isOsaResultsReady()) { - boolean osaThresholdExceeded = ShragaUtils.isThresholdExceeded(config, null, osaResults, new StringBuilder()); - templateData.put("osaThresholdExceeded", osaThresholdExceeded); - buildFailed |= osaThresholdExceeded; - if (config.getEnablePolicyViolations() && !osaResults.getOsaViolations().isEmpty()){ + if (config.getDependencyScannerType() == DependencyScannerType.OSA) { + if (osaResults!=null && osaResults.isOsaResultsReady()) { + boolean thresholdExceeded = scanSummary.isOsaThresholdExceeded(); + templateData.put("osaThresholdExceeded", thresholdExceeded); + buildFailed |= thresholdExceeded; + + //calculate osa bars: + OSASummaryResults osaSummaryResults = osaResults.getResults(); + int osaHigh = osaSummaryResults.getTotalHighVulnerabilities(); + int osaMedium = osaSummaryResults.getTotalMediumVulnerabilities(); + int osaLow = osaSummaryResults.getTotalLowVulnerabilities(); + float osaMaxCount = Math.max(osaHigh, Math.max(osaMedium, osaLow)); + float osaBarNorm = osaMaxCount * 10f / 9f; + + float osaHighTotalHeight = (float) osaHigh / osaBarNorm * 238f; + float osaMediumTotalHeight = (float) osaMedium / osaBarNorm * 238f; + float osaLowTotalHeight = (float) osaLow / osaBarNorm * 238f; + + templateData.put("osaHighTotalHeight", osaHighTotalHeight); + templateData.put("osaMediumTotalHeight", osaMediumTotalHeight); + templateData.put("osaLowTotalHeight", osaLowTotalHeight); + } else { + buildFailed = true; + } + } else if (config.getDependencyScannerType() == DependencyScannerType.SCA){ + boolean thresholdExceeded = scanSummary.isOsaThresholdExceeded(); + templateData.put("scaThresholdExceeded", thresholdExceeded); + buildFailed |= thresholdExceeded; + + //calculate sca bars: + AstScaSummaryResults scaSummaryResults = scaResults.getSummary(); + int scaHigh = scaSummaryResults.getHighVulnerabilityCount(); + int scaMedium = scaSummaryResults.getMediumVulnerabilityCount(); + int scaLow = scaSummaryResults.getLowVulnerabilityCount(); + float scaMaxCount = Math.max(scaHigh, Math.max(scaMedium, scaLow)); + float scaBarNorm = scaMaxCount * 10f / 9f; + + float scaHighTotalHeight = (float) scaHigh / scaBarNorm * 238f; + float scaMediumTotalHeight = (float) scaMedium / scaBarNorm * 238f; + float scaLowTotalHeight = (float) scaLow / scaBarNorm * 238f; + + templateData.put("scaHighTotalHeight", scaHighTotalHeight); + templateData.put("scaMediumTotalHeight", scaMediumTotalHeight); + templateData.put("scaLowTotalHeight", scaLowTotalHeight); + }else{ + buildFailed = true; + } +*/ + + if (config.isOsaEnabled() || config.isAstScaEnabled()) { + if (dependencyScanResult != null && dependencyScanResult.isResultReady()) { + boolean thresholdExceeded = scanSummary.isOsaThresholdExceeded(); + templateData.put("dependencyThresholdExceeded", thresholdExceeded); + if (config.isSastEnabled()) { + buildFailed |= thresholdExceeded || buildFailed; + } else { + buildFailed |= thresholdExceeded; + } + + //calculate dependency results bars: + int dependencyHigh = dependencyScanResult.getHighVulnerability(); + int dependencyMedium = dependencyScanResult.getMediumVulnerability(); + int dependencyLow = dependencyScanResult.getLowVulnerability(); + float dependencyMaxCount = Math.max(dependencyHigh, Math.max(dependencyMedium, dependencyLow)); + float dependencyBarNorm = dependencyMaxCount * 10f / 9f; + + + float dependencyHighTotalHeight = (float) dependencyHigh / dependencyBarNorm * 238f; + float dependencyMediumTotalHeight = (float) dependencyMedium / dependencyBarNorm * 238f; + float dependencyLowTotalHeight = (float) dependencyLow / dependencyBarNorm * 238f; + + templateData.put("dependencyHighTotalHeight", dependencyHighTotalHeight); + templateData.put("dependencyMediumTotalHeight", dependencyMediumTotalHeight); + templateData.put("dependencyLowTotalHeight", dependencyLowTotalHeight); + } else { + buildFailed = true; + } + } + + + if (config.getEnablePolicyViolations()) { + Map policies = new HashMap<>(); + + if (config.isSastEnabled() && sastResults != null && !sastResults.getSastPolicies().isEmpty()) { policyViolated = true; + policies = sastResults.getSastPolicies().stream().collect( + Collectors.toMap(Policy::getPolicyName, + Policy::getRuleName, + (left, right) -> left + )); } - //calculate osa bars: - OSASummaryResults osaSummaryResults = osaResults.getResults(); - int osaHigh = osaSummaryResults.getTotalHighVulnerabilities(); - int osaMedium = osaSummaryResults.getTotalMediumVulnerabilities(); - int osaLow = osaSummaryResults.getTotalLowVulnerabilities(); - float osaMaxCount = Math.max(osaHigh, Math.max(osaMedium, osaLow)); - float osaBarNorm = osaMaxCount * 10f / 9f; - - float osaHighTotalHeight = (float) osaHigh / osaBarNorm * 238f; - float osaMediumTotalHeight = (float) osaMedium / osaBarNorm * 238f; - float osaLowTotalHeight = (float) osaLow / osaBarNorm * 238f; - - templateData.put("osaHighTotalHeight", osaHighTotalHeight); - templateData.put("osaMediumTotalHeight", osaMediumTotalHeight); - templateData.put("osaLowTotalHeight", osaLowTotalHeight); + + if (Boolean.TRUE.equals(config.isOsaEnabled()) + && osaResults != null + && osaResults.getOsaPolicies() != null + && !osaResults.getOsaPolicies().isEmpty()) { + policyViolated = true; + policies.putAll(osaResults.getOsaPolicies().stream().collect( + Collectors.toMap(Policy::getPolicyName, Policy::getRuleName, + (left, right) -> left))); + } + + policyViolatedCount = policies.size(); + String policyLabel = policyViolatedCount == 1 ? "Policy" : "Policies"; + templateData.put("policyLabel", policyLabel); + templateData.put("policyViolatedCount", policyViolatedCount); } - String policyLabel = osaResults.getOsaPolicies().size() == 1? "Policy": "Policies"; - templateData.put("policyLabel", policyLabel); templateData.put("policyViolated", policyViolated); buildFailed |= policyViolated; @@ -109,6 +209,18 @@ public static String generateSummary(SASTResults sastResults, OSAResults osaResu return writer.toString(); } + private static DependencyScanResult resolveDependencyResult(OSAResults osaResults, AstScaResults scaResults) { + DependencyScanResult dependencyScanResult; + if (osaResults != null) { + dependencyScanResult = new DependencyScanResult(osaResults); + } else if (scaResults != null) { + dependencyScanResult = new DependencyScanResult(scaResults); + } else { + dependencyScanResult = null; + } + return dependencyScanResult; + } + private static float calculateNewBarHeight(int newCount, int count, float totalHeight) { int minimalVisibilityHeight = 5; //new high diff --git a/src/main/java/com/cx/restclient/configuration/CxScanConfig.java b/src/main/java/com/cx/restclient/configuration/CxScanConfig.java index 7e70e169..b547226c 100644 --- a/src/main/java/com/cx/restclient/configuration/CxScanConfig.java +++ b/src/main/java/com/cx/restclient/configuration/CxScanConfig.java @@ -1,29 +1,43 @@ package com.cx.restclient.configuration; +import com.cx.restclient.ast.dto.sast.AstSastConfig; +import com.cx.restclient.ast.dto.sca.AstScaConfig; +import com.cx.restclient.dto.*; +import com.cx.restclient.sast.dto.ReportType; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.cookie.Cookie; import java.io.File; import java.io.Serializable; -import java.util.Properties; +import java.util.*; /** * Created by galn on 21/12/2016. */ public class CxScanConfig implements Serializable { - private Boolean sastEnabled = false; - private Boolean osaEnabled = false; - private String cxOrigin; + private String cxOriginUrl; + private CxVersion cxVersion; + private boolean disableCertificateValidation = false; + private boolean useSSOLogin = false; + private String sourceDir; + private String osaLocationPath; private File reportsDir; + // Map / (e.g. PDF to its file path) + private Map reports = new HashMap<>(); private String username; private String password; + private String refreshToken; private String url; private String projectName; private String teamPath; + private String mvnPath; private String teamId; private Boolean denyProject = false; + private Boolean hideResults = false; private Boolean isPublic = true; private Boolean forceScan = false; private String presetName; @@ -31,6 +45,7 @@ public class CxScanConfig implements Serializable { private String sastFolderExclusions; private String sastFilterPattern; private Integer sastScanTimeoutInMinutes; + private Integer osaScanTimeoutInMinutes; private String scanComment; private Boolean isIncremental = false; private Boolean isSynchronous = false; @@ -40,12 +55,22 @@ public class CxScanConfig implements Serializable { private Integer sastLowThreshold; private Boolean sastNewResultsThresholdEnabled = false; private String sastNewResultsThresholdSeverity; - + private TokenLoginResponse token; private Boolean generatePDFReport = false; private File zipFile; - private Integer engineConfigurationId = 1; + private Integer engineConfigurationId; + private String engineConfigurationName; private String osaFolderExclusions; + + public String getEngineConfigurationName() { + return engineConfigurationName; + } + + public void setEngineConfigurationName(String engineConfigurationName) { + this.engineConfigurationName = engineConfigurationName; + } + private String osaFilterPattern; private String osaArchiveIncludePatterns; private Boolean osaGenerateJsonReport = true; @@ -56,10 +81,41 @@ public class CxScanConfig implements Serializable { private Integer osaLowThreshold; private Properties osaFsaConfig; //for MAVEN private String osaDependenciesJson; - + private Boolean avoidDuplicateProjectScans = false; private boolean enablePolicyViolations = false; + private Boolean generateXmlReport = true; private String cxARMUrl; + private String[] paths; + //remote source control + private RemoteSourceTypes remoteType = null; + private String remoteSrcUser; + private String remoteSrcPass; + private String remoteSrcUrl; + private int remoteSrcPort; + private byte[] remoteSrcKeyFile; + private String remoteSrcBranch; + private String perforceMode; + + // CLI config properties + private Integer progressInterval; + private Integer osaProgressInterval; + private Integer connectionRetries; + private String osaScanDepth; + private Integer maxZipSize; + private String defaultProjectName; + + private String scaJsonReport; + + private AstScaConfig astScaConfig; + private AstSastConfig astSastConfig; + + private final Set scannerTypes = new HashSet<>(); + private final List sessionCookies = new ArrayList<>(); + private Boolean isProxy = true; + private ProxyConfig proxyConfig; + private Boolean useNTLM = false; + public CxScanConfig() { } @@ -72,30 +128,63 @@ public CxScanConfig(String url, String username, String password, String cxOrigi this.disableCertificateValidation = disableCertificateValidation; } - public Boolean getSastEnabled() { - return sastEnabled; + public CxScanConfig(String url, String username, String password, String cxOrigin, String cxOriginUrl, boolean disableCertificateValidation) { + this.url = url; + this.username = username; + this.password = password; + this.cxOrigin = cxOrigin; + this.cxOriginUrl = cxOriginUrl; + this.disableCertificateValidation = disableCertificateValidation; } - public void setSastEnabled(Boolean sastEnabled) { - this.sastEnabled = sastEnabled; + + public CxScanConfig(String url, String refreshToken, String cxOrigin, boolean disableCertificateValidation) { + this.url = url; + this.refreshToken = refreshToken; + this.cxOrigin = cxOrigin; + this.disableCertificateValidation = disableCertificateValidation; } - public Boolean getOsaEnabled() { - return osaEnabled; + public boolean isSastEnabled() { + return scannerTypes.contains(ScannerType.SAST); } - public void setOsaEnabled(Boolean osaEnabled) { - this.osaEnabled = osaEnabled; + public boolean isOsaEnabled() { + return scannerTypes.contains(ScannerType.OSA); + } + + public boolean isAstScaEnabled() { + return scannerTypes.contains(ScannerType.AST_SCA); + } + + public boolean isAstSastEnabled() { + return scannerTypes.contains(ScannerType.AST_SAST); + } + + public void setSastEnabled(boolean sastEnabled) { + if (sastEnabled) { + scannerTypes.add(ScannerType.SAST); + } else { + scannerTypes.remove(ScannerType.SAST); + } } public String getCxOrigin() { return cxOrigin; } + public String getCxOriginUrl() { + return cxOriginUrl; + } + public void setCxOrigin(String cxOrigin) { this.cxOrigin = cxOrigin; } + public void setCxOriginUrl(String cxOriginUrl) { + this.cxOriginUrl = cxOriginUrl; + } + public boolean isDisableCertificateValidation() { return disableCertificateValidation; } @@ -104,6 +193,18 @@ public void setDisableCertificateValidation(boolean disableCertificateValidation this.disableCertificateValidation = disableCertificateValidation; } + public boolean isUseSSOLogin() { + return useSSOLogin; + } + + public void setUseSSOLogin(boolean useSSOLogin) { + this.useSSOLogin = useSSOLogin; + } + + public Boolean getAvoidDuplicateProjectScans() { + return avoidDuplicateProjectScans; + } + public String getSourceDir() { return sourceDir; } @@ -112,6 +213,18 @@ public void setSourceDir(String sourceDir) { this.sourceDir = sourceDir; } + public String getOsaLocationPath() { + return osaLocationPath; + } + + public void setOsaLocationPath(String osaLocationPath) { + this.osaLocationPath = osaLocationPath; + } + + public String getEffectiveSourceDirForDependencyScan() { + return osaLocationPath != null ? osaLocationPath : sourceDir; + } + public File getReportsDir() { return reportsDir; } @@ -128,6 +241,14 @@ public void setUsername(String username) { this.username = username; } + public void setRefreshToken(String token) { + this.refreshToken = token; + } + + public String getRefreshToken() { + return refreshToken; + } + public String getPassword() { return password; } @@ -140,6 +261,14 @@ public String getUrl() { return url; } + public String getScaJsonReport() { + return scaJsonReport; + } + + public void setScaJsonReport(String scaJsonReport) { + this.scaJsonReport = scaJsonReport; + } + public void setUrl(String url) { this.url = url; } @@ -157,6 +286,13 @@ public String getTeamPath() { } public void setTeamPath(String teamPath) { + //Make teampath always in the form /CxServer/Team1. User might have used '\' in the path. + if (!StringUtils.isEmpty(teamPath) && !teamPath.startsWith("\\") && !teamPath.startsWith(("/"))) { + teamPath = "/" + teamPath; + } + if (!StringUtils.isEmpty(teamPath) && teamPath != null) { + teamPath = teamPath.replace("\\", "/"); + } this.teamPath = teamPath; } @@ -232,6 +368,14 @@ public void setSastScanTimeoutInMinutes(Integer sastScanTimeoutInMinutes) { this.sastScanTimeoutInMinutes = sastScanTimeoutInMinutes; } + public Integer getOsaScanTimeoutInMinutes() { + return osaScanTimeoutInMinutes == null ? -1 : osaScanTimeoutInMinutes; + } + + public void setOsaScanTimeoutInMinutes(Integer sastOsaScanTimeoutInMinutes) { + this.osaScanTimeoutInMinutes = sastOsaScanTimeoutInMinutes; + } + public String getScanComment() { return scanComment; } @@ -405,11 +549,13 @@ public String getOsaDependenciesJson() { } public boolean isSASTThresholdEffectivelyEnabled() { - return getSastEnabled() && getSastThresholdsEnabled() && (getSastHighThreshold() != null || getSastMediumThreshold() != null || getSastLowThreshold() != null); + return isSastEnabled() && getSastThresholdsEnabled() && (getSastHighThreshold() != null || getSastMediumThreshold() != null || getSastLowThreshold() != null); } public boolean isOSAThresholdEffectivelyEnabled() { - return getOsaEnabled() && getOsaThresholdsEnabled() && (getOsaHighThreshold() != null || getOsaMediumThreshold() != null || getOsaLowThreshold() != null); + return (isOsaEnabled() || isAstScaEnabled()) && + getOsaThresholdsEnabled() && + (getOsaHighThreshold() != null || getOsaMediumThreshold() != null || getOsaLowThreshold() != null); } public void setOsaDependenciesJson(String osaDependenciesJson) { @@ -443,4 +589,258 @@ public String getCxARMUrl() { public void setCxARMUrl(String cxARMUrl) { this.cxARMUrl = cxARMUrl; } + + public Boolean getHideResults() { + return hideResults; + } + + public void setHideResults(Boolean hideResults) { + this.hideResults = hideResults; + } + + + public Boolean isAvoidDuplicateProjectScans() { + return avoidDuplicateProjectScans; + } + + public void setAvoidDuplicateProjectScans(Boolean avoidDuplicateProjectScans) { + this.avoidDuplicateProjectScans = avoidDuplicateProjectScans; + } + + public String getRemoteSrcUser() { + return remoteSrcUser; + } + + public void setRemoteSrcUser(String remoteSrcUser) { + this.remoteSrcUser = remoteSrcUser; + } + + public String getRemoteSrcPass() { + return remoteSrcPass; + } + + public void setRemoteSrcPass(String remoteSrcPass) { + this.remoteSrcPass = remoteSrcPass; + } + + public String getRemoteSrcUrl() { + return remoteSrcUrl; + } + + public void setRemoteSrcUrl(String remoteSrcUrl) { + this.remoteSrcUrl = remoteSrcUrl; + } + + public int getRemoteSrcPort() { + return remoteSrcPort; + } + + public void setRemoteSrcPort(int remoteSrcPort) { + this.remoteSrcPort = remoteSrcPort; + } + + public byte[] getRemoteSrcKeyFile() { + return remoteSrcKeyFile; + } + + public void setRemoteSrcKeyFile(byte[] remoteSrcKeyFile) { + this.remoteSrcKeyFile = remoteSrcKeyFile; + } + + public RemoteSourceTypes getRemoteType() { + return remoteType; + } + + public void setRemoteType(RemoteSourceTypes remoteType) { + this.remoteType = remoteType; + } + + public String[] getPaths() { + return paths; + } + + public void setPaths(String[] paths) { + this.paths = paths; + } + + public String getRemoteSrcBranch() { + return remoteSrcBranch; + } + + public void setRemoteSrcBranch(String remoteSrcBranch) { + this.remoteSrcBranch = remoteSrcBranch; + } + + public String getPerforceMode() { + return perforceMode; + } + + public void setPerforceMode(String perforceMode) { + this.perforceMode = perforceMode; + } + + public Boolean getGenerateXmlReport() { + return generateXmlReport; + } + + public void setGenerateXmlReport(Boolean generateXmlReport) { + this.generateXmlReport = generateXmlReport; + } + + public CxVersion getCxVersion() { + return cxVersion; + } + + public void setCxVersion(CxVersion cxVersion) { + this.cxVersion = cxVersion; + } + + public Integer getProgressInterval() { + return progressInterval; + } + + public void setProgressInterval(Integer progressInterval) { + this.progressInterval = progressInterval; + } + + public Integer getOsaProgressInterval() { + return osaProgressInterval; + } + + public void setOsaProgressInterval(Integer osaProgressInterval) { + this.osaProgressInterval = osaProgressInterval; + } + + public Integer getConnectionRetries() { + return connectionRetries; + } + + public void setConnectionRetries(Integer connectionRetries) { + this.connectionRetries = connectionRetries; + } + + public String getMvnPath() { + return mvnPath; + } + + public void setMvnPath(String mvnPath) { + this.mvnPath = mvnPath; + } + + public String getOsaScanDepth() { + return osaScanDepth; + } + + public void setOsaScanDepth(String osaScanDepth) { + this.osaScanDepth = osaScanDepth; + } + + public Integer getMaxZipSize() { + return maxZipSize; + } + + public void setMaxZipSize(Integer maxZipSize) { + this.maxZipSize = maxZipSize; + } + + public String getDefaultProjectName() { + return defaultProjectName; + } + + public void setDefaultProjectName(String defaultProjectName) { + this.defaultProjectName = defaultProjectName; + } + + public Map getReports() { + return reports; + } + + public void addPDFReport(String pdfReportPath) { + reports.put(ReportType.PDF, pdfReportPath); + } + + public void addXMLReport(String xmlReportPath) { + reports.put(ReportType.XML, xmlReportPath); + } + + public void addCSVReport(String csvReportPath) { + reports.put(ReportType.CSV, csvReportPath); + } + + public void addRTFReport(String rtfReportPath) { + reports.put(ReportType.RTF, rtfReportPath); + } + + public AstScaConfig getAstScaConfig() { + return astScaConfig; + } + + public void setAstScaConfig(AstScaConfig astScaConfig) { + this.astScaConfig = astScaConfig; + } + + public AstSastConfig getAstSastConfig() { + return astSastConfig; + } + + public void setAstSastConfig(AstSastConfig astConfig) { + this.astSastConfig = astConfig; + } + + public Set getScannerTypes() { + return scannerTypes; + } + + public void addScannerType(ScannerType scannerType) { + this.scannerTypes.add(scannerType); + } + + /** + * SAST and OSA are currently deployed on-premises, whereas AST-SCA is deployed in a cloud. + * If SAST or OSA are enabled, some of the config properties are mandatory (url, username, password etc). + * Otherwise, these properties are optional. + */ + public boolean isSastOrOSAEnabled() { + return isSastEnabled() || isOsaEnabled(); + } + + public Boolean isProxy() { + return isProxy; + } + + public void setProxy(Boolean proxy) { + isProxy = proxy; + } + + public ProxyConfig getProxyConfig() { + return proxyConfig; + } + + public void setProxyConfig(ProxyConfig proxyConfig) { + this.proxyConfig = proxyConfig; + } + + public void addCookie(Cookie cookie) { + this.sessionCookies.add(cookie); + } + + public List getSessionCookie() { + return this.sessionCookies; + } + + public TokenLoginResponse getToken() { + return token; + } + + public void setToken(TokenLoginResponse token) { + this.token = token; + } + + public Boolean getNTLM() { + return useNTLM; + } + + public void setNTLM(Boolean ntlm) { + useNTLM = ntlm; + } } diff --git a/src/main/java/com/cx/restclient/configuration/PropertyFileLoader.java b/src/main/java/com/cx/restclient/configuration/PropertyFileLoader.java new file mode 100644 index 00000000..b357eb35 --- /dev/null +++ b/src/main/java/com/cx/restclient/configuration/PropertyFileLoader.java @@ -0,0 +1,58 @@ +package com.cx.restclient.configuration; + +import com.cx.restclient.exception.CxClientException; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +@Slf4j +public class PropertyFileLoader { + private static final String DEFAULT_FILENAME = "common.properties"; + + @Getter(lazy = true) + private static final PropertyFileLoader defaultInstance = new PropertyFileLoader(DEFAULT_FILENAME); + + private final Properties properties; + + /** + * Loads properties from resources. + * + * @param filenames list of resource filenames to load properties from. + * If the same property appears several times in the files, the property value from a file will be overridden with the value from the next file. + * @throws CxClientException if no filenames is provided, or if an error occurred while loading a file. + */ + public PropertyFileLoader(String... filenames) { + if (filenames.length == 0) { + throw new CxClientException("Please provide at least one filename."); + } + + properties = new Properties(); + for (String filename : filenames) { + Properties singleFileProperties = getPropertiesFromResource(filename); + properties.putAll(singleFileProperties); + } + } + + private Properties getPropertiesFromResource(String resourceName) { + Properties result = new Properties(); + + log.debug("Loading properties from resource: {}", resourceName); + try (InputStream input = getClass().getClassLoader().getResourceAsStream(resourceName)) { + if (input != null) { + result.load(input); + } else { + log.warn("Unable to find resource: {}, skipping.", resourceName); + } + } catch (IOException e) { + throw new CxClientException(String.format("Error loading the '%s' resource.", resourceName), e); + } + return result; + } + + public String get(String propertyName) { + return properties.getProperty(propertyName); + } +} diff --git a/src/main/java/com/cx/restclient/cxArm/dto/Policy.java b/src/main/java/com/cx/restclient/cxArm/dto/Policy.java index 3b73a1f0..63b118db 100644 --- a/src/main/java/com/cx/restclient/cxArm/dto/Policy.java +++ b/src/main/java/com/cx/restclient/cxArm/dto/Policy.java @@ -13,7 +13,21 @@ public class Policy implements Serializable { long policyId; String policyName; - List violations = new ArrayList(); + String ruleName; + String firstDetectionDate; + private List violations = new ArrayList(); + + public Policy() { + } + + + public Policy(long policyId, String policyName, String ruleName, List violations, String firstDate) { + this.policyId = policyId; + this.policyName = policyName; + this.ruleName = ruleName; + this.violations = violations; + this.firstDetectionDate = firstDate; + } public long getPolicyId() { return policyId; @@ -36,11 +50,22 @@ public List getViolations() { } public void setViolations(List violations) { - //Add the policy name to the violations for the report - for(Violation violation: violations){ - violation.setPolicyName(policyName); - } - this.violations = violations; } + + public String getRuleName() { + return ruleName; + } + + public void setRuleName(String ruleName) { + this.ruleName = ruleName; + } + + public String getFirstDetectionDate() { + return firstDetectionDate; + } + + public void setFirstDetectionDate(String firstDetectionDate) { + this.firstDetectionDate = firstDetectionDate; + } } diff --git a/src/main/java/com/cx/restclient/cxArm/dto/Rule.java b/src/main/java/com/cx/restclient/cxArm/dto/Rule.java new file mode 100644 index 00000000..d0028a8c --- /dev/null +++ b/src/main/java/com/cx/restclient/cxArm/dto/Rule.java @@ -0,0 +1,19 @@ +package com.cx.restclient.cxArm.dto; + +import java.util.ArrayList; +import java.util.List; + +/** + * Created by Galn on 11/11/2018. + */ +public class Rule { + List violations = new ArrayList(); + + public List getViolations() { + return violations; + } + + public void setViolations(List violations) { + this.violations = violations; + } +} diff --git a/src/main/java/com/cx/restclient/cxArm/dto/Violation.java b/src/main/java/com/cx/restclient/cxArm/dto/Violation.java index ced4e617..4e18a82a 100644 --- a/src/main/java/com/cx/restclient/cxArm/dto/Violation.java +++ b/src/main/java/com/cx/restclient/cxArm/dto/Violation.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import java.io.Serializable; import java.util.Date; import static com.cx.restclient.common.ShragaUtils.formatDate; @@ -10,7 +11,7 @@ * Created by Galn on 7/5/2018. */ @JsonIgnoreProperties(ignoreUnknown = true) -public class Violation { +public class Violation implements Serializable { private String ruleId; private String ruleName; @@ -47,9 +48,6 @@ public class Violation { private String policyName; - private String detectionDate; - - public String getRuleId() { return ruleId; } @@ -193,16 +191,4 @@ public String getPolicyName() { public void setPolicyName(String policyName) { this.policyName = policyName; } - - public String getDetectionDate() { - if (detectionDate == null){ - String date = new Date(firstDetectionDateByArm).toString(); - detectionDate = formatDate(date, "E MMM dd hh:mm:ss Z yyyy", "dd/MM/yy"); - } - return detectionDate; - } - - public void setDetectionDate(String detectionDate) { - this.detectionDate = detectionDate; - } } diff --git a/src/main/java/com/cx/restclient/cxArm/utils/CxARMUtils.java b/src/main/java/com/cx/restclient/cxArm/utils/CxARMUtils.java index 569814f4..fa416ba4 100644 --- a/src/main/java/com/cx/restclient/cxArm/utils/CxARMUtils.java +++ b/src/main/java/com/cx/restclient/cxArm/utils/CxARMUtils.java @@ -1,21 +1,75 @@ package com.cx.restclient.cxArm.utils; import com.cx.restclient.cxArm.dto.Policy; +import com.cx.restclient.cxArm.dto.Violation; import com.cx.restclient.exception.CxClientException; import com.cx.restclient.httpClient.CxHttpClient; import java.io.IOException; -import java.util.List; +import java.util.*; +import java.util.stream.Collectors; import static com.cx.restclient.common.CxPARAM.CX_ARM_VIOLATION; +import static com.cx.restclient.common.ShragaUtils.formatDate; import static com.cx.restclient.httpClient.utils.ContentType.CONTENT_TYPE_APPLICATION_JSON_V1; /** * Created by Galn on 7/30/2018. */ public abstract class CxARMUtils { - public static List getProjectViolations(CxHttpClient httpClient, String cxARMUrl, long projectId, String provider) throws IOException, CxClientException { + public static List getProjectViolatedPolicies(CxHttpClient httpClient, String cxARMUrl, long projectId, String provider) throws IOException, CxClientException { String relativePath = CX_ARM_VIOLATION.replace("{projectId}", Long.toString(projectId)).replace("{provider}", provider); return (List) httpClient.getRequest(cxARMUrl, relativePath, CONTENT_TYPE_APPLICATION_JSON_V1, null, Policy.class, 200, "CxARM violations", true); } + + public static List getPolicyList(Policy policy) { + List policies = new ArrayList(); + Map> rules = resolveRules(policy.getViolations()); + Iterator it = rules.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry pair = (Map.Entry) it.next(); + List violations = (List) pair.getValue(); + String firstDate = resolveFirstDate(violations); + policies.add(new Policy(policy.getPolicyId(), policy.getPolicyName(), pair.getKey().toString(), violations, firstDate)); + it.remove(); // avoids a ConcurrentModificationException + } + + return policies; + } + + private static Map> resolveRules(List violations) { + Map> rules = violations.stream().collect( + Collectors.toMap(Violation::getRuleName, e -> { + List ary = new ArrayList(); + ary.add(e); + return ary; + }, + (left, right) -> { + left.addAll(right); + return left; + })); + + return rules; + } + + private static String resolveFirstDate(List violations) { + Date firstDetectionDate = new Date(violations.get(0).getFirstDetectionDateByArm()); + for (Violation violation : violations) { + Date date = new Date(violation.getFirstDetectionDateByArm()); + if (date.before(firstDetectionDate)) { + firstDetectionDate = date; + } + } + String firstDate = formatDate(firstDetectionDate.toString(), "E MMM dd hh:mm:ss Z yyyy", "dd/MM/yy"); + return firstDate; + } + + public static String getPoliciesNames(List policies) { + String str =""; + for (Policy policy : policies){ + str += ", " + policy.getPolicyName(); + } + str = str.substring(1, str.length()); + return str; + } } diff --git a/src/main/java/com/cx/restclient/dto/BaseStatus.java b/src/main/java/com/cx/restclient/dto/BaseStatus.java index b9376e68..8d853702 100644 --- a/src/main/java/com/cx/restclient/dto/BaseStatus.java +++ b/src/main/java/com/cx/restclient/dto/BaseStatus.java @@ -1,8 +1,11 @@ package com.cx.restclient.dto; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + /** * Created by Galn on 13/02/2018. */ +@JsonIgnoreProperties(ignoreUnknown = true) public class BaseStatus { private String baseId; private Status baseStatus; diff --git a/src/main/java/com/cx/restclient/dto/CxProxy.java b/src/main/java/com/cx/restclient/dto/CxProxy.java new file mode 100644 index 00000000..59084b84 --- /dev/null +++ b/src/main/java/com/cx/restclient/dto/CxProxy.java @@ -0,0 +1,60 @@ +package com.cx.restclient.dto; + +/** + * Created by Galn on 4/2/2019. + */ +public class CxProxy { + private Boolean useProxy; + private String proxyHost; + private Integer proxyPort; + private String proxyScheme; + + + public CxProxy(Boolean useProxy, String proxyHost, Integer proxyPort, String scheme) { + this.useProxy = useProxy; + this.proxyHost = proxyHost; + this.proxyPort = proxyPort; + this.proxyScheme = scheme; + } + + public CxProxy() { + } + + public Boolean getUseProxy() { + return useProxy; + } + + public void setUseProxy(Boolean useProxy) { + this.useProxy = useProxy; + } + + public String getProxyHost() { + return proxyHost; + } + + public void setProxyHost(String proxyHost) { + this.proxyHost = proxyHost; + } + + public Integer getProxyPort() { + return proxyPort; + } + + public void setProxyPort(Integer proxyPort) { + this.proxyPort = proxyPort; + } + + public void setProxyPort(String proxyPort) { + try { + this.proxyPort = Integer.parseInt(proxyPort); + }catch (Exception ex){} + } + + public String getProxyScheme() { + return proxyScheme; + } + + public void setProxyScheme(String proxyScheme) { + this.proxyScheme = proxyScheme; + } +} diff --git a/src/main/java/com/cx/restclient/dto/CxVersion.java b/src/main/java/com/cx/restclient/dto/CxVersion.java new file mode 100644 index 00000000..6d1d2d6e --- /dev/null +++ b/src/main/java/com/cx/restclient/dto/CxVersion.java @@ -0,0 +1,28 @@ +package com.cx.restclient.dto; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +/** + * Created by Galn on 4/1/2019. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class CxVersion { + private String version; + private String hotFix; + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String getHotFix() { + return hotFix; + } + + public void setHotFix(String hotFix) { + this.hotFix = hotFix; + } +} diff --git a/src/main/java/com/cx/restclient/dto/EngineConfiguration.java b/src/main/java/com/cx/restclient/dto/EngineConfiguration.java new file mode 100644 index 00000000..e3f9d1eb --- /dev/null +++ b/src/main/java/com/cx/restclient/dto/EngineConfiguration.java @@ -0,0 +1,31 @@ +package com.cx.restclient.dto; + +public class EngineConfiguration { + + private int id; + + private String name; + + public EngineConfiguration() { + } + + public EngineConfiguration(String name) { + this.name = name; + } + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } +} diff --git a/src/main/java/com/cx/restclient/dto/LoginRequest.java b/src/main/java/com/cx/restclient/dto/LoginRequest.java index cbf9e413..abfa1cc5 100644 --- a/src/main/java/com/cx/restclient/dto/LoginRequest.java +++ b/src/main/java/com/cx/restclient/dto/LoginRequest.java @@ -1,9 +1,12 @@ package com.cx.restclient.dto; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + /** * Created by: Dorg. * Date: 08/09/2016. */ +@JsonIgnoreProperties(ignoreUnknown = true) public class LoginRequest { private String username; diff --git a/src/main/java/com/cx/restclient/dto/LoginSettings.java b/src/main/java/com/cx/restclient/dto/LoginSettings.java new file mode 100644 index 00000000..e134c448 --- /dev/null +++ b/src/main/java/com/cx/restclient/dto/LoginSettings.java @@ -0,0 +1,96 @@ +package com.cx.restclient.dto; + +import com.cx.restclient.osa.dto.ClientType; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.NoArgsConstructor; +import org.apache.http.cookie.Cookie; + +import java.util.ArrayList; +import java.util.List; + +@NoArgsConstructor +@AllArgsConstructor +@Builder +public class LoginSettings { + private String accessControlBaseUrl; + private String username; + private String password; + private CharSequence tenant; + private String refreshToken; + private final List sessionCookies = new ArrayList<>(); + private String version; + + // TODO: find a way to use a single client type here. + private ClientType clientTypeForRefreshToken; + private ClientType clientTypeForPasswordAuth; + + public void setAccessControlBaseUrl(String accessControlBaseUrl) { + this.accessControlBaseUrl = accessControlBaseUrl; + } + + public String getAccessControlBaseUrl() { + return accessControlBaseUrl; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getUsername() { + return username; + } + + public void setPassword(String password) { + this.password = password; + } + + public String getPassword() { + return password; + } + + public void setClientTypeForRefreshToken(ClientType clientType) { + this.clientTypeForRefreshToken = clientType; + } + + public ClientType getClientTypeForRefreshToken() { + return clientTypeForRefreshToken; + } + + public void setRefreshToken(String refreshToken) { + this.refreshToken = refreshToken; + } + + public String getRefreshToken() { + return refreshToken; + } + + public void setClientTypeForPasswordAuth(ClientType clientType) { + this.clientTypeForPasswordAuth = clientType; + } + + public ClientType getClientTypeForPasswordAuth() { + return clientTypeForPasswordAuth; + } + + public CharSequence getTenant() { + return tenant; + } + + public void setTenant(CharSequence tenant) { + this.tenant = tenant; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public List getSessionCookies() { + return sessionCookies; + } + +} diff --git a/src/main/java/com/cx/restclient/dto/PathFilter.java b/src/main/java/com/cx/restclient/dto/PathFilter.java new file mode 100644 index 00000000..b982ccc7 --- /dev/null +++ b/src/main/java/com/cx/restclient/dto/PathFilter.java @@ -0,0 +1,35 @@ +package com.cx.restclient.dto; + +import com.cx.restclient.common.ShragaUtils; +import org.apache.commons.lang3.ArrayUtils; +import org.slf4j.Logger; + +import java.util.List; +import java.util.Map; + +public class PathFilter { + private String[] includes; + private String[] excludes; + + public PathFilter(String folderExclusions, String filterPattern, Logger log) { + Map> stringListMap = ShragaUtils.generateIncludesExcludesPatternLists(folderExclusions, filterPattern, log); + includes = getArray(stringListMap, ShragaUtils.INCLUDES_LIST); + excludes = getArray(stringListMap, ShragaUtils.EXCLUDES_LIST); + } + + public String[] getIncludes() { + return includes; + } + + public String[] getExcludes() { + return excludes; + } + + private static String[] getArray(Map> map, String key){ + return map.get(key).toArray(new String[0]); + } + + public void addToIncludes(String element) { + includes = ArrayUtils.add(includes, element); + } +} diff --git a/src/main/java/com/cx/restclient/dto/ProxyConfig.java b/src/main/java/com/cx/restclient/dto/ProxyConfig.java new file mode 100644 index 00000000..39a1d9f4 --- /dev/null +++ b/src/main/java/com/cx/restclient/dto/ProxyConfig.java @@ -0,0 +1,87 @@ +package com.cx.restclient.dto; + +import java.io.Serializable; + +public class ProxyConfig implements Serializable { + + private String host; + private int port; + private String username; + private String password; + private boolean useHttps; + private String noproxyHosts; + + public String getNoproxyHosts() { + return noproxyHosts; + } + + public void setNoproxyHosts(String noproxyHosts) { + this.noproxyHosts = noproxyHosts; + } + + public ProxyConfig() { + } + + public ProxyConfig(String host, int port, String username, String password, boolean useHttps) { + this.host = host; + this.port = port; + this.username = username; + this.password = password; + this.useHttps = useHttps; + this.noproxyHosts = ""; + } + /** + * TO be used in Bamboo plugin for the time being + * @param host + * @param port + * @param username + * @param password + * @param useHttps + */ + public ProxyConfig(String host, int port, String username, String password, boolean useHttps, String noProxyHosts){ + this.port = port; + this.username = username; + this.password = password; + this.useHttps = useHttps; + this.noproxyHosts = noProxyHosts; + } + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public int getPort() { + return port; + } + + public void setPort(int port) { + this.port = port; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public boolean isUseHttps() { + return useHttps; + } + + public void setUseHttps(boolean useHttps) { + this.useHttps = useHttps; + } +} diff --git a/src/main/java/com/cx/restclient/dto/RemoteSourceRequest.java b/src/main/java/com/cx/restclient/dto/RemoteSourceRequest.java new file mode 100644 index 00000000..23f755b9 --- /dev/null +++ b/src/main/java/com/cx/restclient/dto/RemoteSourceRequest.java @@ -0,0 +1,131 @@ +package com.cx.restclient.dto; + +import com.cx.restclient.configuration.CxScanConfig; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; + +/** + * Created by Galn on 11/25/2018. + */ +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonIgnoreProperties({ "type" }) +public class RemoteSourceRequest { + + public class Credentials { + private String userName; + private String password; + + public Credentials(String userName, String password) { + this.userName = userName; + this.password = password; + } + + public String getUserName() { + return userName; + } + + public void setUserName(String userName) { + this.userName = userName; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + } + + public class Uri { + private String absoluteUrl; + private int port; + + public Uri(String absoluteUrl, int port) { + this.absoluteUrl = absoluteUrl; + this.port = port; + } + + public String getAbsoluteUrl() { + return absoluteUrl; + } + + public void setAbsoluteUrl(String absoluteUrl) { + this.absoluteUrl = absoluteUrl; + } + + public int getPort() { + return port; + } + + public void setPort(int port) { + this.port = port; + } + } + + private Credentials credentials; + private Uri uri; + private byte[] privateKey; + private String[] paths; + private RemoteSourceTypes type; + private String browseMode; + + public RemoteSourceRequest() { + } + + public RemoteSourceRequest(CxScanConfig config) { + credentials = new Credentials(config.getRemoteSrcUser(), config.getRemoteSrcPass()); + uri = new Uri(config.getRemoteSrcUrl(), config.getRemoteSrcPort()); + privateKey = config.getRemoteSrcKeyFile() == null ? new byte[0] : config.getRemoteSrcKeyFile(); + paths = config.getPaths(); + type = config.getRemoteType(); + } + + public Credentials getCredentials() { + return credentials; + } + + public void setCredentials(Credentials credentials) { + this.credentials = credentials; + } + + public Uri getUri() { + return uri; + } + + public void setUri(Uri uri) { + this.uri = uri; + } + + public byte[] getPrivateKey() { + return privateKey; + } + + public void setPrivateKey(byte[] privateKey) { + this.privateKey = privateKey; + } + + public String[] getPaths() { + return paths; + } + + public void setPaths(String[] paths) { + this.paths = paths; + } + + public RemoteSourceTypes getType() { + return type; + } + + public void setType(RemoteSourceTypes type) { + this.type = type; + } + + public String getBrowseMode() { + return browseMode; + } + + public void setBrowseMode(String browseMode) { + this.browseMode = browseMode; + } +} diff --git a/src/main/java/com/cx/restclient/dto/RemoteSourceTypes.java b/src/main/java/com/cx/restclient/dto/RemoteSourceTypes.java new file mode 100644 index 00000000..02f07277 --- /dev/null +++ b/src/main/java/com/cx/restclient/dto/RemoteSourceTypes.java @@ -0,0 +1,25 @@ +package com.cx.restclient.dto; + +/** + * Created by: Galn. + * Date: 25/11/2016. + */ +public enum RemoteSourceTypes { + + SHARED("shared"), + SVN("svn"), + GIT("git"), + TFS("tfs"), + PERFORCE("perforce"); + + private String value; + + RemoteSourceTypes(String value) { + this.value = value; + } + + public String value() { + return value; + } + +} \ No newline at end of file diff --git a/src/main/java/com/cx/restclient/dto/Results.java b/src/main/java/com/cx/restclient/dto/Results.java new file mode 100644 index 00000000..6a1f7fbb --- /dev/null +++ b/src/main/java/com/cx/restclient/dto/Results.java @@ -0,0 +1,11 @@ +package com.cx.restclient.dto; + +import com.cx.restclient.exception.CxClientException; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public abstract class Results { + private CxClientException exception; +} diff --git a/src/main/java/com/cx/restclient/dto/ScanResults.java b/src/main/java/com/cx/restclient/dto/ScanResults.java index c4d67649..98d49480 100644 --- a/src/main/java/com/cx/restclient/dto/ScanResults.java +++ b/src/main/java/com/cx/restclient/dto/ScanResults.java @@ -1,69 +1,65 @@ package com.cx.restclient.dto; +import com.cx.restclient.ast.dto.sast.AstSastResults; +import com.cx.restclient.ast.dto.sca.AstScaResults; +import com.cx.restclient.exception.CxClientException; import com.cx.restclient.osa.dto.OSAResults; import com.cx.restclient.sast.dto.SASTResults; import java.io.Serializable; +import java.util.EnumMap; +import java.util.Map; -public class ScanResults implements Serializable { +public class ScanResults extends Results implements Serializable { + private final Map resultsMap = new EnumMap<>(ScannerType.class); - private SASTResults sastResults; - private OSAResults osaResults; + private Exception generalException = null; - private Exception sastCreateException = null; - private Exception sastWaitException = null; - private Exception osaCreateException = null; - private Exception osaWaitException = null; - - public ScanResults() { + public Map getResults(){ + return resultsMap; } - - public SASTResults getSastResults() { - return sastResults; + + public void put(ScannerType type, Results results) { + if(resultsMap.containsKey(type)){ + throw new CxClientException("Results already contain type " + type); + } + resultsMap.put(type, results); } - public void setSastResults(SASTResults sastResults) { - this.sastResults = sastResults; + public Map getResultsMap() { + return resultsMap; } - public OSAResults getOsaResults() { - return osaResults; + public Results get(ScannerType type) { + return resultsMap.get(type); } - public void setOsaResults(OSAResults osaResults) { - this.osaResults = osaResults; - } - public Exception getSastCreateException() { - return sastCreateException; + public OSAResults getOsaResults() { + return (OSAResults)resultsMap.get(ScannerType.OSA); } - public void setSastCreateException(Exception sastCreateException) { - this.sastCreateException = sastCreateException; + public AstSastResults getAstResults() { + return (AstSastResults)resultsMap.get(ScannerType.AST_SAST); } - public Exception getSastWaitException() { - return sastWaitException; + public AstScaResults getScaResults() { + return (AstScaResults)resultsMap.get(ScannerType.AST_SCA); } - public void setSastWaitException(Exception sastWaitException) { - this.sastWaitException = sastWaitException; - } - public Exception getOsaCreateException() { - return osaCreateException; - } + public SASTResults getSastResults() { + return (SASTResults)resultsMap.get(ScannerType.SAST); - public void setOsaCreateException(Exception osaCreateException) { - this.osaCreateException = osaCreateException; } - public Exception getOsaWaitException() { - return osaWaitException; + public Exception getGeneralException() { + return generalException; } - public void setOsaWaitException(Exception osaWaitException) { - this.osaWaitException = osaWaitException; + public void setGeneralException(Exception generalException) { + this.generalException = generalException; } + } diff --git a/src/main/java/com/cx/restclient/dto/ScannerType.java b/src/main/java/com/cx/restclient/dto/ScannerType.java new file mode 100644 index 00000000..f69a19e1 --- /dev/null +++ b/src/main/java/com/cx/restclient/dto/ScannerType.java @@ -0,0 +1,21 @@ +package com.cx.restclient.dto; + +public enum ScannerType { + // Legacy scanners. + SAST("CxSAST"), + OSA("CxOSA"), + + // Scan engines of the new CxAST platform. + AST_SCA("CxAST-SCA"), + AST_SAST("CxAST-SAST"); + + private final String displayName; + + ScannerType(String displayName) { + this.displayName = displayName; + } + + public String getDisplayName() { + return displayName; + } +} diff --git a/src/main/java/com/cx/restclient/dto/SourceLocationType.java b/src/main/java/com/cx/restclient/dto/SourceLocationType.java new file mode 100644 index 00000000..2e2e94e4 --- /dev/null +++ b/src/main/java/com/cx/restclient/dto/SourceLocationType.java @@ -0,0 +1,16 @@ +package com.cx.restclient.dto; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +@Getter +public enum SourceLocationType { + LOCAL_DIRECTORY("upload"), + REMOTE_REPOSITORY("git"); + + /** + * Value used in API calls. Currently all the clients using this enum use the same API values. + */ + private final String apiValue; +} diff --git a/src/main/java/com/cx/restclient/dto/Team.java b/src/main/java/com/cx/restclient/dto/Team.java index 8af500c4..2f389cdd 100644 --- a/src/main/java/com/cx/restclient/dto/Team.java +++ b/src/main/java/com/cx/restclient/dto/Team.java @@ -1,8 +1,12 @@ package com.cx.restclient.dto; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + /** * Created by Galn on 14/02/2018. */ + +@JsonIgnoreProperties(ignoreUnknown = true) public class Team { public String id; public String fullName; diff --git a/src/main/java/com/cx/restclient/dto/ThresholdResult.java b/src/main/java/com/cx/restclient/dto/ThresholdResult.java deleted file mode 100644 index 35671989..00000000 --- a/src/main/java/com/cx/restclient/dto/ThresholdResult.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.cx.restclient.dto; - -/** - * Created by Galn on 4/10/2018. - */ -public class ThresholdResult { - private boolean isFail; - private String failDescription; - - public ThresholdResult(boolean isFail, String failDescription) { - this.isFail = isFail; - this.failDescription = failDescription; - } - - public boolean isFail() { - return isFail; - } - - public void setFail(boolean fail) { - isFail = fail; - } - - public String getFailDescription() { - return failDescription; - } - - public void setFailDescription(String failDescription) { - this.failDescription = failDescription; - } -} diff --git a/src/main/java/com/cx/restclient/dto/TokenLoginResponse.java b/src/main/java/com/cx/restclient/dto/TokenLoginResponse.java index ffec9c1b..948aa620 100644 --- a/src/main/java/com/cx/restclient/dto/TokenLoginResponse.java +++ b/src/main/java/com/cx/restclient/dto/TokenLoginResponse.java @@ -1,12 +1,16 @@ package com.cx.restclient.dto; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + /** * Created by Galn on 19/03/2018. */ +@JsonIgnoreProperties(ignoreUnknown = true) public class TokenLoginResponse { private String access_token; private long expires_in; private String token_type; + private String refresh_token; public String getAccess_token() { return access_token; @@ -31,4 +35,12 @@ public String getToken_type() { public void setToken_type(String token_type) { this.token_type = token_type; } + + public String getRefresh_token() { + return refresh_token; + } + + public void setRefresh_token(String refresh_token) { + this.refresh_token = refresh_token; + } } diff --git a/src/main/java/com/cx/restclient/dto/scansummary/ErrorSource.java b/src/main/java/com/cx/restclient/dto/scansummary/ErrorSource.java new file mode 100644 index 00000000..501e5dfc --- /dev/null +++ b/src/main/java/com/cx/restclient/dto/scansummary/ErrorSource.java @@ -0,0 +1,7 @@ +package com.cx.restclient.dto.scansummary; + +public enum ErrorSource { + SAST, + OSA, + SCA +} diff --git a/src/main/java/com/cx/restclient/dto/scansummary/ScanSummary.java b/src/main/java/com/cx/restclient/dto/scansummary/ScanSummary.java new file mode 100644 index 00000000..5bfe90fc --- /dev/null +++ b/src/main/java/com/cx/restclient/dto/scansummary/ScanSummary.java @@ -0,0 +1,161 @@ +package com.cx.restclient.dto.scansummary; + +import com.cx.restclient.ast.dto.sca.AstScaResults; +import com.cx.restclient.ast.dto.sca.report.AstScaSummaryResults; +import com.cx.restclient.common.CxPARAM; +import com.cx.restclient.configuration.CxScanConfig; +import com.cx.restclient.osa.dto.OSAResults; +import com.cx.restclient.osa.dto.OSASummaryResults; +import com.cx.restclient.sast.dto.SASTResults; + +import java.util.ArrayList; +import java.util.List; + +/** + * Collects errors from provided scan results, based on scan config. + */ +public class ScanSummary { + private final List thresholdErrors = new ArrayList<>(); + private final List newResultThresholdErrors = new ArrayList<>(); + private final boolean policyViolated; + + public ScanSummary(CxScanConfig config, SASTResults sastResults, OSAResults osaResults, AstScaResults scaResults) { + + addSastThresholdErrors(config, sastResults); + addDependencyScanThresholdErrors(config, osaResults, scaResults); + + addNewResultThresholdErrors(config, sastResults); + + policyViolated = determinePolicyViolation(config, sastResults, osaResults, scaResults); + } + + @Override + public String toString() { + StringBuilder result = new StringBuilder(); + + for (ThresholdError error : thresholdErrors) { + result.append(String.format("%s %s severity results are above threshold. Results: %d. Threshold: %d.%n", + error.getSource().toString(), + error.getSeverity().toString().toLowerCase(), + error.getValue(), + error.getThreshold())); + } + + for (Severity severity : newResultThresholdErrors) { + result.append(String.format("One or more new results of %s severity%n", severity.toString().toLowerCase())); + } + + if (policyViolated) { + result.append(CxPARAM.PROJECT_POLICY_VIOLATED_STATUS).append("\n"); + } + + return result.toString(); + } + + public List getThresholdErrors() { + return thresholdErrors; + } + + public boolean hasErrors() { + return !thresholdErrors.isEmpty() || !newResultThresholdErrors.isEmpty() || policyViolated; + } + + public boolean isPolicyViolated() { + return policyViolated; + } + + public boolean isSastThresholdExceeded() { + return thresholdErrors.stream().anyMatch(error -> error.getSource() == ErrorSource.SAST); + } + + public boolean isOsaThresholdExceeded() { + return thresholdErrors.stream().anyMatch(error -> error.getSource() == ErrorSource.OSA || error.getSource() == ErrorSource.SCA); + } + + public boolean isSastThresholdForNewResultsExceeded() { + return !newResultThresholdErrors.isEmpty(); + } + + private void addSastThresholdErrors(CxScanConfig config, SASTResults sastResults) { + if (config.isSASTThresholdEffectivelyEnabled() && + sastResults != null && + sastResults.isSastResultsReady()) { + checkForThresholdError(sastResults.getHigh(), config.getSastHighThreshold(), ErrorSource.SAST, Severity.HIGH); + checkForThresholdError(sastResults.getMedium(), config.getSastMediumThreshold(), ErrorSource.SAST, Severity.MEDIUM); + checkForThresholdError(sastResults.getLow(), config.getSastLowThreshold(), ErrorSource.SAST, Severity.LOW); + } + } + + private void addDependencyScanThresholdErrors(CxScanConfig config, OSAResults osaResults, AstScaResults scaResults) { + if (config.isOSAThresholdEffectivelyEnabled() && (scaResults != null || osaResults != null)) { + + ErrorSource errorSource = osaResults != null ? ErrorSource.OSA : ErrorSource.SCA; + int totalHigh = 0; + int totalMedium = 0; + int totalLow = 0; + boolean hasSummary = false; + + if (scaResults != null) { + AstScaSummaryResults summary = scaResults.getSummary(); + if (summary != null) { + hasSummary = true; + totalHigh = summary.getHighVulnerabilityCount(); + totalMedium = summary.getMediumVulnerabilityCount(); + totalLow = summary.getLowVulnerabilityCount(); + } + } else if (osaResults.isOsaResultsReady()) { + OSASummaryResults summary = osaResults.getResults(); + if (summary != null) { + hasSummary = true; + totalHigh = summary.getTotalHighVulnerabilities(); + totalMedium = summary.getTotalMediumVulnerabilities(); + totalLow = summary.getTotalLowVulnerabilities(); + } + } + + if (hasSummary) { + checkForThresholdError(totalHigh, config.getOsaHighThreshold(), errorSource, Severity.HIGH); + checkForThresholdError(totalMedium, config.getOsaMediumThreshold(), errorSource, Severity.MEDIUM); + checkForThresholdError(totalLow, config.getOsaLowThreshold(), errorSource, Severity.LOW); + } + } + } + + private void addNewResultThresholdErrors(CxScanConfig config, SASTResults sastResults) { + if (sastResults != null && sastResults.isSastResultsReady() && config.getSastNewResultsThresholdEnabled()) { + String severity = config.getSastNewResultsThresholdSeverity(); + + if ("LOW".equals(severity)) { + if (sastResults.getNewLow() > 0) { + newResultThresholdErrors.add(Severity.LOW); + } + severity = "MEDIUM"; + } + + if ("MEDIUM".equals(severity)) { + if (sastResults.getNewMedium() > 0) { + newResultThresholdErrors.add(Severity.MEDIUM); + } + severity = "HIGH"; + } + + if ("HIGH".equals(severity) && sastResults.getNewHigh() > 0) { + newResultThresholdErrors.add(Severity.HIGH); + } + } + } + + private static boolean determinePolicyViolation(CxScanConfig config, SASTResults sastResults, OSAResults osaResults, AstScaResults scaResults) { + return config.getEnablePolicyViolations() && + ((osaResults != null && + !osaResults.getOsaPolicies().isEmpty()) || + (sastResults != null && !sastResults.getSastPolicies().isEmpty())) || (scaResults != null && scaResults.isBreakTheBuild() && scaResults.isPolicyViolated()); + } + + private void checkForThresholdError(int value, Integer threshold, ErrorSource source, Severity severity) { + if (threshold != null && value > threshold) { + ThresholdError error = new ThresholdError(source, severity, value, threshold); + thresholdErrors.add(error); + } + } +} diff --git a/src/main/java/com/cx/restclient/dto/scansummary/Severity.java b/src/main/java/com/cx/restclient/dto/scansummary/Severity.java new file mode 100644 index 00000000..5773df9e --- /dev/null +++ b/src/main/java/com/cx/restclient/dto/scansummary/Severity.java @@ -0,0 +1,7 @@ +package com.cx.restclient.dto.scansummary; + +public enum Severity { + LOW, + MEDIUM, + HIGH +} diff --git a/src/main/java/com/cx/restclient/dto/scansummary/ThresholdError.java b/src/main/java/com/cx/restclient/dto/scansummary/ThresholdError.java new file mode 100644 index 00000000..3b002ed8 --- /dev/null +++ b/src/main/java/com/cx/restclient/dto/scansummary/ThresholdError.java @@ -0,0 +1,31 @@ +package com.cx.restclient.dto.scansummary; + +public class ThresholdError { + private ErrorSource source; + private Severity severity; + private final int value; + private final Integer threshold; + + public ThresholdError(ErrorSource source, Severity severity, int value, Integer threshold) { + this.source = source; + this.severity = severity; + this.value = value; + this.threshold = threshold; + } + + public ErrorSource getSource() { + return source; + } + + public Severity getSeverity() { + return severity; + } + + public int getValue() { + return value; + } + + public Integer getThreshold() { + return threshold; + } +} diff --git a/src/main/java/com/cx/restclient/exception/CxClientException.java b/src/main/java/com/cx/restclient/exception/CxClientException.java index 6284f00e..df27ac27 100644 --- a/src/main/java/com/cx/restclient/exception/CxClientException.java +++ b/src/main/java/com/cx/restclient/exception/CxClientException.java @@ -3,7 +3,7 @@ /** * Created by Galn on 05/02/2018. */ -public class CxClientException extends Exception { +public class CxClientException extends RuntimeException { public CxClientException() { super(); } diff --git a/src/main/java/com/cx/restclient/exception/CxHTTPClientException.java b/src/main/java/com/cx/restclient/exception/CxHTTPClientException.java index 8103a5e4..f33ed730 100644 --- a/src/main/java/com/cx/restclient/exception/CxHTTPClientException.java +++ b/src/main/java/com/cx/restclient/exception/CxHTTPClientException.java @@ -4,12 +4,13 @@ * Created by Galn on 05/02/2018. */ public class CxHTTPClientException extends CxClientException { - private int statusCode = 0; + private String responseBody; - public CxHTTPClientException(int statusCode, String s) { - super(s); + public CxHTTPClientException(int statusCode, String message, String responseBody) { + super(message); this.statusCode = statusCode; + this.responseBody = responseBody; } public CxHTTPClientException() { @@ -32,5 +33,7 @@ public int getStatusCode() { return this.statusCode; } - + public String getResponseBody() { + return responseBody; + } } diff --git a/src/main/java/com/cx/restclient/httpClient/CxHttpClient.java b/src/main/java/com/cx/restclient/httpClient/CxHttpClient.java index 4c6cc71c..82a8ab8f 100644 --- a/src/main/java/com/cx/restclient/httpClient/CxHttpClient.java +++ b/src/main/java/com/cx/restclient/httpClient/CxHttpClient.java @@ -1,128 +1,622 @@ package com.cx.restclient.httpClient; import com.cx.restclient.common.ErrorMessage; -import com.cx.restclient.common.UrlUtils; +import com.cx.restclient.dto.LoginSettings; +import com.cx.restclient.dto.ProxyConfig; import com.cx.restclient.dto.TokenLoginResponse; import com.cx.restclient.exception.CxClientException; import com.cx.restclient.exception.CxHTTPClientException; import com.cx.restclient.exception.CxTokenExpiredException; +import com.cx.restclient.osa.dto.ClientType; +import com.google.gson.Gson; +import org.apache.commons.lang3.StringUtils; import org.apache.http.*; +import org.apache.http.auth.AuthSchemeProvider; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.NTCredentials; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CookieStore; +import org.apache.http.client.CredentialsProvider; import org.apache.http.client.HttpClient; +import org.apache.http.client.config.AuthSchemes; +import org.apache.http.client.config.CookieSpecs; +import org.apache.http.client.config.RequestConfig; import org.apache.http.client.entity.UrlEncodedFormEntity; import org.apache.http.client.methods.*; +import org.apache.http.client.params.AuthPolicy; import org.apache.http.client.utils.HttpClientUtils; +import org.apache.http.config.Registry; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.conn.routing.HttpRoute; +import org.apache.http.conn.socket.ConnectionSocketFactory; +import org.apache.http.conn.socket.PlainConnectionSocketFactory; import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.conn.ssl.TrustAllStrategy; +import org.apache.http.conn.ssl.TrustSelfSignedStrategy; +import org.apache.http.cookie.Cookie; import org.apache.http.entity.ContentType; -import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.NoConnectionReuseStrategy; +import org.apache.http.impl.auth.BasicSchemeFactory; +import org.apache.http.impl.auth.DigestSchemeFactory; +import org.apache.http.impl.auth.win.WindowsCredentialsProvider; +import org.apache.http.impl.auth.win.WindowsNTLMSchemeFactory; +import org.apache.http.impl.auth.win.WindowsNegotiateSchemeFactory; +import org.apache.http.impl.client.*; +import org.apache.http.impl.conn.DefaultProxyRoutePlanner; +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; +import org.apache.http.message.BasicHeader; import org.apache.http.message.BasicNameValuePair; import org.apache.http.protocol.HttpContext; +import org.apache.http.ssl.SSLContextBuilder; import org.apache.http.ssl.SSLContexts; import org.apache.http.ssl.TrustStrategy; import org.slf4j.Logger; +import javax.annotation.Nullable; +import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.SSLContext; +import java.io.Closeable; import java.io.IOException; import java.io.UnsupportedEncodingException; -import java.net.MalformedURLException; import java.net.UnknownHostException; +import java.nio.charset.StandardCharsets; import java.security.KeyManagementException; import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; -import java.security.cert.CertificateException; -import java.security.cert.X509Certificate; -import java.util.ArrayList; -import java.util.List; +import java.text.MessageFormat; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; -import static com.cx.restclient.common.CxPARAM.AUTHENTICATION; -import static com.cx.restclient.common.CxPARAM.ORIGIN_HEADER; +import static com.cx.restclient.common.CxPARAM.*; import static com.cx.restclient.httpClient.utils.ContentType.CONTENT_TYPE_APPLICATION_JSON; import static com.cx.restclient.httpClient.utils.HttpClientHelper.*; + /** * Created by Galn on 05/02/2018. */ -public class CxHttpClient { +public class CxHttpClient implements Closeable { + + private static String HTTP_NO_HOST = System.getProperty("http.nonProxyHosts"); + private static String HTTPS_NO_HOST = System.getProperty("https.nonProxyHosts"); + + private static final String HTTPS = "https"; + + private static final String LOGIN_FAILED_MSG = "Fail to login with windows authentication: "; + + private static final String DEFAULT_GRANT_TYPE = "password"; + private static final String LOCATION_HEADER = "Location"; + private static final String AUTH_MESSAGE = "authenticate"; + private static final String CLIENT_SECRET_PROP = "client_secret"; + public static final String REFRESH_TOKEN_PROP = "refresh_token"; + private static final String PASSWORD_PROP = "password"; + public static final String CLIENT_ID_PROP = "client_id"; + private static final String KEY_USER = "user"; + private static final String KEY_DOMAIN = "domain"; - private Logger logi; private HttpClient apacheClient; + + private Logger log; private TokenLoginResponse token; private String rootUri; - private final String username; - private final String password; + private final String refreshToken; private String cxOrigin; + private String cxOriginUrl; + private Boolean useSSo; + private Boolean useNTLM; + private LoginSettings lastLoginSettings; + private String teamPath; + private CookieStore cookieStore = new BasicCookieStore(); + private HttpClientBuilder cb = HttpClients.custom(); + private final Map customHeaders = new HashMap<>(); - private final HttpRequestInterceptor requestFilter = new HttpRequestInterceptor() { - public void process(HttpRequest httpRequest, HttpContext httpContext) throws HttpException, IOException { - httpRequest.addHeader(ORIGIN_HEADER, cxOrigin); - if (token != null) { - httpRequest.addHeader(HttpHeaders.AUTHORIZATION, token.getToken_type() + " " + token.getAccess_token()); - } - } - }; - public CxHttpClient(String hostname, String username, String password, String origin, boolean disableSSLValidation, Logger logi) throws MalformedURLException { - this.logi = logi; - this.username = username; - this.password = password; - this.rootUri = UrlUtils.parseURLToString(hostname, "CxRestAPI/"); + public CxHttpClient(String rootUri, String origin, boolean disableSSLValidation, boolean isSSO, String refreshToken, + boolean isProxy, @Nullable ProxyConfig proxyConfig, Logger log, Boolean useNTLM) throws CxClientException { + this.log = log; + this.rootUri = rootUri; + this.refreshToken = refreshToken; this.cxOrigin = origin; + this.useSSo = isSSO; + this.useNTLM = useNTLM; //create httpclient - HttpClientBuilder builder = HttpClientBuilder.create().addInterceptorFirst(requestFilter); + cb.setDefaultRequestConfig(RequestConfig.custom().setCookieSpec(CookieSpecs.STANDARD).build()); + setSSLTls("TLSv1.2", log); + SSLContextBuilder builder = new SSLContextBuilder(); + SSLConnectionSocketFactory sslConnectionSocketFactory = null; + Registry registry; + PoolingHttpClientConnectionManager cm = null; + if (disableSSLValidation) { + try { + builder.loadTrustMaterial(null, new TrustSelfSignedStrategy()); + sslConnectionSocketFactory = new SSLConnectionSocketFactory(builder.build(), NoopHostnameVerifier.INSTANCE); + registry = RegistryBuilder.create() + .register("http", new PlainConnectionSocketFactory()) + .register(HTTPS, sslConnectionSocketFactory) + .build(); + cm = new PoolingHttpClientConnectionManager(registry); + cm.setMaxTotal(100); + } catch (NoSuchAlgorithmException | KeyStoreException | KeyManagementException e) { + log.error(e.getMessage()); + } + cb.setSSLSocketFactory(sslConnectionSocketFactory); + cb.setConnectionManager(cm); + } else { + cb.setConnectionManager(getHttpConnectionManager(false)); + } + cb.setConnectionManagerShared(true); + + if (isProxy) { + if (!setCustomProxy(cb, proxyConfig, log)) { + cb.useSystemProperties(); + } + } + + if (Boolean.TRUE.equals(useSSo)) { + cb.setDefaultCredentialsProvider(new WindowsCredentialsProvider(new SystemDefaultCredentialsProvider())); + cb.setDefaultCookieStore(cookieStore); + } else { + cb.setConnectionReuseStrategy(new NoConnectionReuseStrategy()); + } + cb.setDefaultAuthSchemeRegistry(getAuthSchemeProviderRegistry()); + + if (useNTLM) { + setNTLMProxy(proxyConfig, cb, log); + } else apacheClient = cb.build(); + } + + public CxHttpClient(String rootUri, String origin, String originUrl, boolean disableSSLValidation, boolean isSSO, String refreshToken, + boolean isProxy, @Nullable ProxyConfig proxyConfig, Logger log, Boolean useNTLM) throws CxClientException { + this(rootUri, origin, disableSSLValidation, isSSO, refreshToken, isProxy, proxyConfig, log, useNTLM); + this.cxOriginUrl = originUrl; + } + + public void setRootUri(String rootUri) { + this.rootUri = rootUri; + } + + public String getRootUri() { + return rootUri; + } + + private void setNTLMProxy(ProxyConfig proxyConfig, HttpClientBuilder cb, Logger log) { + + if (proxyConfig == null || + StringUtils.isEmpty(proxyConfig.getHost()) || + proxyConfig.getPort() == 0) { + log.info("Proxy configuration not provided."); + apacheClient = cb.build(); + return; + } + + log.info("Setting NTLM proxy for Checkmarx http client"); + HttpHost proxy = new HttpHost(proxyConfig.getHost(), proxyConfig.getPort(), "http"); + + HashMap userDomainMap = splitDomainAndTheUserName(proxyConfig.getUsername()); + String user = userDomainMap.get(KEY_USER); + String domain = userDomainMap.get(KEY_DOMAIN); + + CredentialsProvider credsProvider = new BasicCredentialsProvider(); + final NTCredentials credentials = new NTCredentials(user, proxyConfig.getPassword(), null, domain); + credsProvider.setCredentials(AuthScope.ANY, credentials); + + apacheClient = HttpClientBuilder.create() + .setDefaultCredentialsProvider(credsProvider) + .setProxy(proxy) + .setProxyAuthenticationStrategy(ProxyAuthenticationStrategy.INSTANCE) + .setDefaultRequestConfig(RequestConfig.custom() + .setAuthenticationEnabled(true) + .setProxyPreferredAuthSchemes(Arrays.asList(AuthPolicy.NTLM)) + .build() + ) + .build(); + } + + private static HashMap splitDomainAndTheUserName(String userName) { + String domain = ""; + String user = ""; + // If the username has a backslash, then the domain is the first part and the username is the second part + if (userName.contains("\\")) { + String[] parts = userName.split("[\\\\]"); + if (parts.length == 2) { + domain = parts[0]; + user = parts[1]; + } + } else if (userName.contains("/")) { + // If the username has a slash, then the domain is the first part and the username is the second part + String[] parts = userName.split("[/]"); + if (parts.length == 2) { + domain = parts[0]; + user = parts[1]; + } + } else if (userName.contains("@")) { + // If the username has an asterisk, then the domain is the second part and the username is the first part + String[] parts = userName.split("[@]"); + if (parts.length == 2) { + user = parts[0]; + domain = parts[1]; + } + } + + HashMap userDomain = new HashMap(); + userDomain.put(KEY_USER, user); + userDomain.put(KEY_DOMAIN, domain); + return userDomain; + } + + private static boolean setCustomProxy(HttpClientBuilder cb, ProxyConfig proxyConfig, Logger logi) { + if (proxyConfig == null || + StringUtils.isEmpty(proxyConfig.getHost()) || + proxyConfig.getPort() == 0) { + return false; + } + + String scheme = proxyConfig.isUseHttps() ? HTTPS : "http"; + HttpHost proxy = new HttpHost(proxyConfig.getHost(), proxyConfig.getPort(), scheme); + if (StringUtils.isNotEmpty(proxyConfig.getUsername()) && + StringUtils.isNotEmpty(proxyConfig.getPassword())) { + UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(proxyConfig.getUsername(), proxyConfig.getPassword()); + CredentialsProvider credsProvider = new BasicCredentialsProvider(); + credsProvider.setCredentials(new AuthScope(proxy), credentials); + cb.setDefaultCredentialsProvider(credsProvider); + } + + logi.info("Setting proxy for Checkmarx http client"); + cb.setProxy(proxy); + cb.setRoutePlanner(getRoutePlanner(proxyConfig, proxy, logi)); + cb.setProxyAuthenticationStrategy(new ProxyAuthenticationStrategy()); + return true; + } + + private static DefaultProxyRoutePlanner getRoutePlanner(ProxyConfig proxyConfig, HttpHost proxyHost, Logger logi) { + return new DefaultProxyRoutePlanner(proxyHost) { + public HttpRoute determineRoute( + final HttpHost host, + final HttpRequest request, + final HttpContext context) throws HttpException { + String hostname = host.getHostName(); + String noHost = proxyConfig.getNoproxyHosts(); // StringUtils.isNotEmpty(HTTP_NO_HOST) ? HTTP_NO_HOST : HTTPS_NO_HOST; + if (StringUtils.isNotEmpty(noHost)) { + String[] hosts = noHost.split("\\|"); + for (String nonHost : hosts) { + try { + if (matchNonProxyHostWildCard(hostname, noHost)) { + logi.debug("Bypassing proxy as host " + hostname + " is found in the nonProxyHosts"); + return new HttpRoute(host); + } + } catch (PatternSyntaxException e) { + logi.warn("Wrong nonProxyHost param: " + nonHost); + } + } + } + return super.determineRoute(host, request, context); + } + }; + } + + /* + * '*' is the only wildcard support in nonProxyHosts JVM argument. + * * in Java regex has different meaning than required here. + * Hence the custom logic + */ + private static boolean matchNonProxyHostWildCard(String sourceHost, String nonProxyHost) { + if (nonProxyHost.indexOf("*") > -1) + nonProxyHost = nonProxyHost.replaceAll("\\.", "\\\\."); + + nonProxyHost = nonProxyHost.replaceAll("\\*", "\\.\\*"); + + Pattern p = Pattern.compile(nonProxyHost);//. represents single character + Matcher m = p.matcher(sourceHost); + return m.matches(); + } + + + private static SSLConnectionSocketFactory getTrustAllSSLSocketFactory() { + TrustStrategy acceptingTrustStrategy = new TrustAllStrategy(); + SSLContext sslContext; + try { + sslContext = SSLContexts.custom().loadTrustMaterial(null, acceptingTrustStrategy).build(); + } catch (NoSuchAlgorithmException | KeyStoreException | KeyManagementException e) { + throw new CxClientException("Fail to set trust all certificate, 'SSLConnectionSocketFactory'", e); + } + return new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE); + } + + private static PoolingHttpClientConnectionManager getHttpConnectionManager(boolean disableSSLValidation) { + ConnectionSocketFactory factory; if (disableSSLValidation) { - builder = disableCertificateValidation(builder, logi); + factory = getTrustAllSSLSocketFactory(); + } else { + factory = new SSLConnectionSocketFactory(SSLContexts.createDefault()); } - apacheClient = builder.build(); + Registry socketFactoryRegistry = RegistryBuilder.create() + .register(HTTPS, factory) + .register("http", new PlainConnectionSocketFactory()) + .build(); + PoolingHttpClientConnectionManager connManager = new PoolingHttpClientConnectionManager(socketFactoryRegistry); + connManager.setMaxTotal(50); + connManager.setDefaultMaxPerRoute(5); + return connManager; } - public void login() throws IOException, CxClientException { - UrlEncodedFormEntity requestEntity = generateUrlEncodedFormEntity(); - HttpPost post = new HttpPost(rootUri + AUTHENTICATION); - token = request(post, ContentType.APPLICATION_FORM_URLENCODED.toString(), requestEntity, TokenLoginResponse.class, HttpStatus.SC_OK, "authenticate", false, false); + private static Registry getAuthSchemeProviderRegistry() { + return RegistryBuilder.create() + .register(AuthSchemes.DIGEST, new DigestSchemeFactory()) + .register(AuthSchemes.BASIC, new BasicSchemeFactory()) + .register(AuthSchemes.NTLM, new WindowsNTLMSchemeFactory(null)) + .register(AuthSchemes.SPNEGO, new WindowsNegotiateSchemeFactory(null)) + .build(); } - private UrlEncodedFormEntity generateUrlEncodedFormEntity() throws UnsupportedEncodingException { - List parameters = new ArrayList(); - parameters.add(new BasicNameValuePair("username", username)); - parameters.add(new BasicNameValuePair("password", password)); - parameters.add(new BasicNameValuePair("grant_type", "password")); - parameters.add(new BasicNameValuePair("scope", "sast_rest_api cxarm_api")); - parameters.add(new BasicNameValuePair("client_id", "resource_owner_client")); - parameters.add(new BasicNameValuePair("client_secret", "014DF517-39D1-4453-B7B3-9930C563627C")); + public void login(LoginSettings settings) throws IOException { + lastLoginSettings = settings; + + if (!settings.getSessionCookies().isEmpty()) { + setSessionCookies(settings.getSessionCookies()); + return; + } - return new UrlEncodedFormEntity(parameters, "utf-8"); + if (settings.getRefreshToken() != null) { + token = getAccessTokenFromRefreshToken(settings); + } else if (Boolean.TRUE.equals(useSSo)) { + if (settings.getVersion().equals("lower than 9.0")) { + ssoLegacyLogin(); + } else { + token = ssoLogin(); + // Don't delete this print. VS Code plugin relies on CxCLI output to work properly. + // Also we don't want the token to appear in regular logs. + System.out.printf("Access Token: %s%n", token.getAccess_token()); // NOSONAR: we need standard output here. + } + } else { + token = generateToken(settings); + } + } + + public ArrayList ssoLegacyLogin() { + HttpUriRequest request; + HttpResponse loginResponse = null; + + try { + request = RequestBuilder.post() + .setUri(rootUri + "auth/ssologin") + .setConfig(RequestConfig.DEFAULT) + .setEntity(new StringEntity("", StandardCharsets.UTF_8)) + .build(); + + loginResponse = apacheClient.execute(request); + + } catch (IOException e) { + String message = LOGIN_FAILED_MSG + e.getMessage(); + log.error(message); + throw new CxClientException(message); + } finally { + HttpClientUtils.closeQuietly(loginResponse); + } + setSessionCookies(cookieStore.getCookies()); + + //return cookies clone - for IDE's usage + return new ArrayList<>(cookieStore.getCookies()); + } + + private void setSessionCookies(List cookies) { + String cxCookie = null; + String csrfToken = null; + + for (Cookie cookie : cookies) { + if (cookie.getName().equals(CSRF_TOKEN_HEADER)) { + csrfToken = cookie.getValue(); + } + if (cookie.getName().equals("cxCookie")) { + cxCookie = cookie.getValue(); + } + } + + List
headers = new ArrayList<>(); + headers.add(new BasicHeader(CSRF_TOKEN_HEADER, csrfToken)); + headers.add(new BasicHeader("cookie", String.format("CXCSRFToken=%s; cxCookie=%s", csrfToken, cxCookie))); + + // Don't delete these prints, they are being used on VS Code plugin + System.out.println(CSRF_TOKEN_HEADER + ": " + csrfToken); + System.out.printf("cookie: CXCSRFToken=%s; cxCookie=%s%n", csrfToken, cxCookie); + + apacheClient = cb.setDefaultHeaders(headers).build(); + } + + private TokenLoginResponse ssoLogin() { + HttpUriRequest request; + HttpResponse response; + final String BASE_URL = "/auth/identity/"; + + RequestConfig requestConfig = RequestConfig.custom() + .setRedirectsEnabled(false) + .setAuthenticationEnabled(true) + .setCookieSpec(CookieSpecs.STANDARD) + .build(); + try { + //Request1 + request = RequestBuilder.post() + .setUri(rootUri + SSO_AUTHENTICATION) + .setConfig(requestConfig) + .setHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_FORM_URLENCODED.toString()) + .setEntity(generateSSOEntity()) + .build(); + + response = apacheClient.execute(request); + + //Request2 + String cookies = retrieveCookies(); + String redirectURL = response.getHeaders(LOCATION_HEADER)[0].getValue(); + request = RequestBuilder.get() + .setUri(rootUri + BASE_URL + redirectURL) + .setConfig(requestConfig) + .setHeader("Cookie", cookies) + .setHeader("Upgrade-Insecure-Requests", "1") + .build(); + response = apacheClient.execute(request); + + //Request3 + cookies = retrieveCookies(); + redirectURL = response.getHeaders(LOCATION_HEADER)[0].getValue(); + redirectURL = rootUri + redirectURL.replace("/CxRestAPI/", ""); + request = RequestBuilder.get() + .setUri(redirectURL) + .setConfig(requestConfig) + .setHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_FORM_URLENCODED.toString()) + .setHeader("Cookie", cookies) + .build(); + response = apacheClient.execute(request); + return extractToken(response); + } catch (IOException e) { + throw new CxClientException(LOGIN_FAILED_MSG + e.getMessage()); + } + } + + private TokenLoginResponse extractToken(HttpResponse response) { + String redirectURL = response.getHeaders(LOCATION_HEADER)[0].getValue(); + if (!redirectURL.contains("access_token")) { + throw new CxClientException("Failed retrieving access token from server"); + } + return new Gson().fromJson(urlToJson(redirectURL), TokenLoginResponse.class); + } + + private String urlToJson(String url) { + url = url.replace("=", "\":\""); + url = url.replace("&", "\",\""); + return "{\"" + url + "\"}"; + } + + private String retrieveCookies() { + List cookieList = cookieStore.getCookies(); + final StringBuilder builder = new StringBuilder(); + cookieList.forEach(cookie -> + builder.append(cookie.getName()).append("=").append(cookie.getValue()).append(";")); + return builder.toString(); + } + + public TokenLoginResponse generateToken(LoginSettings settings) throws IOException { + UrlEncodedFormEntity requestEntity = getAuthRequest(settings); + HttpPost post = new HttpPost(settings.getAccessControlBaseUrl()); + try { + return request(post, ContentType.APPLICATION_FORM_URLENCODED.toString(), requestEntity, + TokenLoginResponse.class, HttpStatus.SC_OK, AUTH_MESSAGE, false, false); + } catch (CxClientException e) { + if (!e.getMessage().contains("invalid_scope")) { + throw new CxClientException(String.format("Failed to generate access token, failure error was: %s", e.getMessage()), e); + } + ClientType.RESOURCE_OWNER.setScopes("sast_rest_api"); + settings.setClientTypeForPasswordAuth(ClientType.RESOURCE_OWNER); + requestEntity = getAuthRequest(settings); + return request(post, ContentType.APPLICATION_FORM_URLENCODED.toString(), requestEntity, + TokenLoginResponse.class, HttpStatus.SC_OK, AUTH_MESSAGE, false, false); + } + } + + private TokenLoginResponse getAccessTokenFromRefreshToken(LoginSettings settings) throws IOException { + UrlEncodedFormEntity requestEntity = getTokenRefreshingRequest(settings); + HttpPost post = new HttpPost(settings.getAccessControlBaseUrl()); + try { + return request(post, ContentType.APPLICATION_FORM_URLENCODED.toString(), requestEntity, + TokenLoginResponse.class, HttpStatus.SC_OK, AUTH_MESSAGE, false, false); + } catch (CxClientException e) { + throw new CxClientException(String.format("Failed to generate access token from refresh token. The error was: %s", e.getMessage()), e); + } + } + + public void revokeToken(String token) throws IOException { + UrlEncodedFormEntity requestEntity = getRevocationRequest(ClientType.CLI, token); + HttpPost post = new HttpPost(rootUri + REVOCATION); + try { + request(post, ContentType.APPLICATION_FORM_URLENCODED.toString(), requestEntity, + String.class, HttpStatus.SC_OK, "revocation", false, false); + } catch (CxClientException e) { + throw new CxClientException(String.format("Token revocation failure error was: %s", e.getMessage()), e); + } + } + + private static UrlEncodedFormEntity getRevocationRequest(ClientType clientType, String token) { + List parameters = new ArrayList<>(); + parameters.add(new BasicNameValuePair("token_type_hint", REFRESH_TOKEN_PROP)); + parameters.add(new BasicNameValuePair("token", token)); + parameters.add(new BasicNameValuePair(CLIENT_ID_PROP, clientType.getClientId())); + parameters.add(new BasicNameValuePair(CLIENT_SECRET_PROP, clientType.getClientSecret())); + + return new UrlEncodedFormEntity(parameters, StandardCharsets.UTF_8); + } + + private static UrlEncodedFormEntity getAuthRequest(LoginSettings settings) { + ClientType clientType = settings.getClientTypeForPasswordAuth(); + String grantType = StringUtils.defaultString(clientType.getGrantType(), DEFAULT_GRANT_TYPE); + List parameters = new ArrayList<>(); + parameters.add(new BasicNameValuePair("username", settings.getUsername())); + parameters.add(new BasicNameValuePair(PASSWORD_PROP, settings.getPassword())); + parameters.add(new BasicNameValuePair("grant_type", grantType)); + parameters.add(new BasicNameValuePair("scope", clientType.getScopes())); + parameters.add(new BasicNameValuePair(CLIENT_ID_PROP, clientType.getClientId())); + parameters.add(new BasicNameValuePair(CLIENT_SECRET_PROP, clientType.getClientSecret())); + + if (!StringUtils.isEmpty(settings.getTenant())) { + String authContext = String.format("Tenant:%s", settings.getTenant()); + parameters.add(new BasicNameValuePair("acr_values", authContext)); + } + + return new UrlEncodedFormEntity(parameters, StandardCharsets.UTF_8); + } + + private static UrlEncodedFormEntity getTokenRefreshingRequest(LoginSettings settings) throws UnsupportedEncodingException { + ClientType clientType = settings.getClientTypeForRefreshToken(); + List parameters = new ArrayList<>(); + parameters.add(new BasicNameValuePair("grant_type", REFRESH_TOKEN_PROP)); + parameters.add(new BasicNameValuePair(CLIENT_ID_PROP, clientType.getClientId())); + parameters.add(new BasicNameValuePair(CLIENT_SECRET_PROP, clientType.getClientSecret())); + parameters.add(new BasicNameValuePair(REFRESH_TOKEN_PROP, settings.getRefreshToken())); + + return new UrlEncodedFormEntity(parameters, StandardCharsets.UTF_8.name()); } //GET REQUEST - public T getRequest(String relPath, String contentType, Class responseType, int expectStatus, String failedMsg, boolean isCollection) throws IOException, CxClientException { - return getRequest(rootUri, relPath, CONTENT_TYPE_APPLICATION_JSON, contentType, responseType, expectStatus, failedMsg, isCollection); + public T getRequest(String relPath, String contentType, Class responseType, int expectStatus, String failedMsg, boolean isCollection) throws IOException { + return getRequest(rootUri, relPath, CONTENT_TYPE_APPLICATION_JSON, contentType, responseType, expectStatus, failedMsg, isCollection); } - public T getRequest(String rootURL, String relPath, String acceptHeader, String contentType, Class responseType, int expectStatus, String failedMsg, boolean isCollection) throws IOException, CxClientException { + public T getRequest(String rootURL, String relPath, String acceptHeader, String contentType, Class responseType, int expectStatus, String failedMsg, boolean isCollection) throws IOException { HttpGet get = new HttpGet(rootURL + relPath); get.addHeader(HttpHeaders.ACCEPT, acceptHeader); return request(get, contentType, null, responseType, expectStatus, "get " + failedMsg, isCollection, true); } //POST REQUEST - public T postRequest(String relPath, String contentType, HttpEntity entity, Class responseType, int expectStatus, String failedMsg) throws IOException, CxClientException { + public T postRequest(String relPath, String contentType, HttpEntity entity, Class responseType, int expectStatus, String failedMsg) throws IOException { HttpPost post = new HttpPost(rootUri + relPath); return request(post, contentType, entity, responseType, expectStatus, failedMsg, false, true); } //PUT REQUEST - public T putRequest(String relPath, String contentType, HttpEntity entity, Class responseType, int expectStatus, String failedMsg) throws IOException, CxClientException { + public T putRequest(String relPath, String contentType, HttpEntity entity, Class responseType, int expectStatus, String failedMsg) throws IOException { HttpPut put = new HttpPut(rootUri + relPath); return request(put, contentType, entity, responseType, expectStatus, failedMsg, false, true); } //PATCH REQUEST - public void patchRequest(String relPath, String contentType, HttpEntity entity, int expectStatus, String failedMsg) throws IOException, CxClientException { + public void patchRequest(String relPath, String contentType, HttpEntity entity, int expectStatus, String failedMsg) throws IOException { HttpPatch patch = new HttpPatch(rootUri + relPath); request(patch, contentType, entity, null, expectStatus, failedMsg, false, true); } - private T request(HttpRequestBase httpMethod, String contentType, HttpEntity entity, Class responseType, int expectStatus, String failedMsg, boolean isCollection, boolean retry) throws IOException, CxClientException { + public void setTeamPathHeader(String teamPath) { + this.teamPath = teamPath; + } + + public void addCustomHeader(String name, String value) { + log.debug(String.format("Adding a custom header: %s: %s", name, value)); + customHeaders.put(name, value); + } + + private T request(HttpRequestBase httpMethod, String contentType, HttpEntity entity, Class responseType, int expectStatus, String failedMsg, boolean isCollection, boolean retry) throws IOException { if (contentType != null) { httpMethod.addHeader("Content-type", contentType); } @@ -130,24 +624,40 @@ private T request(HttpRequestBase httpMethod, String contentType, HttpEntity ((HttpEntityEnclosingRequestBase) httpMethod).setEntity(entity); } HttpResponse response = null; + int statusCode = 0; try { + httpMethod.addHeader(ORIGIN_HEADER, cxOrigin); + httpMethod.addHeader(ORIGIN_URL_HEADER, cxOriginUrl); + httpMethod.addHeader(TEAM_PATH, this.teamPath); + if (token != null) { + httpMethod.addHeader(HttpHeaders.AUTHORIZATION, token.getToken_type() + " " + token.getAccess_token()); + } + + for (Map.Entry entry : customHeaders.entrySet()) { + httpMethod.addHeader(entry.getKey(), entry.getValue()); + } + response = apacheClient.execute(httpMethod); + statusCode = response.getStatusLine().getStatusCode(); - if (response.getStatusLine().getStatusCode() == HttpStatus.SC_UNAUTHORIZED) { //Token expired + if (statusCode == HttpStatus.SC_UNAUTHORIZED) { // Token has probably expired throw new CxTokenExpiredException(extractResponseBody(response)); } + validateResponse(response, expectStatus, "Failed to " + failedMsg); //extract response as object and return the link return convertToObject(response, responseType, isCollection); - } catch (UnknownHostException e){ + } catch (UnknownHostException e) { throw new CxHTTPClientException(ErrorMessage.CHECKMARX_SERVER_CONNECTION_FAILED.getErrorMessage()); } catch (CxTokenExpiredException ex) { if (retry) { - logi.warn("Access token expired, requesting a new token"); - login(); - return request(httpMethod, contentType, entity, responseType, expectStatus, failedMsg, isCollection, false); + logTokenError(httpMethod, statusCode, ex); + if (lastLoginSettings != null) { + login(lastLoginSettings); + return request(httpMethod, contentType, entity, responseType, expectStatus, failedMsg, isCollection, false); + } } throw ex; } finally { @@ -160,20 +670,57 @@ public void close() { HttpClientUtils.closeQuietly(apacheClient); } - private HttpClientBuilder disableCertificateValidation(HttpClientBuilder builder, Logger logi) { + private void setSSLTls(String protocol, Logger log) { try { - SSLContext disabledSSLContext = SSLContexts.custom().loadTrustMaterial(new TrustStrategy() { - public boolean isTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException { - return true; - } - }).build(); - builder.setSslcontext(disabledSSLContext); - builder.setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE); - } catch (KeyManagementException | NoSuchAlgorithmException | KeyStoreException e) { - logi.warn("Failed to disable certificate verification: " + e.getMessage()); + final SSLContext sslContext = SSLContext.getInstance(protocol); + sslContext.init(null, null, null); + HttpsURLConnection.setDefaultSSLSocketFactory(sslContext.getSocketFactory()); + } catch (NoSuchAlgorithmException | KeyManagementException e) { + log.warn(String.format("Failed to set SSL TLS : %s", e.getMessage())); } + } + + //TODO handle missing scope issue with management_and_orchestration_api + private StringEntity generateSSOEntity() { + final String clientId = "cxsast_client"; + final String redirectUri = "%2Fcxwebclient%2FauthCallback.html%3F"; + final String responseType = "id_token%20token"; + final String nonce = "9313f0902ba64e50bc564f5137f35a52"; + final String isPrompt = "true"; + final String scopes = "sast_api openid sast-permissions access-control-permissions access_control_api management_and_orchestration_api".replace(" ", "%20"); + final String providerId = "2"; //windows provider id + + String redirectUrl = MessageFormat.format("/CxRestAPI/auth/identity/connect/authorize/callback" + + "?client_id={0}" + + "&redirect_uri={1}" + redirectUri + + "&response_type={2}" + + "&scope={3}" + + "&nonce={4}" + + "&prompt={5}" + , clientId, rootUri, responseType, scopes, nonce, isPrompt); + try { + List urlParameters = new ArrayList<>(); + urlParameters.add(new BasicNameValuePair("redirectUrl", redirectUrl)); + urlParameters.add(new BasicNameValuePair("providerid", providerId)); + return new UrlEncodedFormEntity(urlParameters, StandardCharsets.UTF_8.name()); + } catch (UnsupportedEncodingException e) { + throw new CxClientException(e.getMessage()); + } + } + + public void setToken(TokenLoginResponse token) { + this.token = token; + } + + private void logTokenError(HttpRequestBase httpMethod, int statusCode, CxTokenExpiredException ex) { + String message = String.format("Received status code %d for URL: %s with the message: %s", + statusCode, + httpMethod.getURI(), + ex.getMessage()); + + log.warn(message); - return builder; + log.info("Possible reason: access token has expired. Trying to request a new token..."); } } diff --git a/src/main/java/com/cx/restclient/httpClient/utils/ContentType.java b/src/main/java/com/cx/restclient/httpClient/utils/ContentType.java index de4a4616..7fca6c17 100644 --- a/src/main/java/com/cx/restclient/httpClient/utils/ContentType.java +++ b/src/main/java/com/cx/restclient/httpClient/utils/ContentType.java @@ -5,6 +5,8 @@ */ public class ContentType { public static final String CONTENT_TYPE_APPLICATION_JSON = "application/json"; + public static final String CONTENT_TYPE_APPLICATION_JSON_V21 = "application/json;v=2.1"; + public static final String CONTENT_TYPE_APPLICATION_JSON_V2 = "application/json;v=2.0"; public static final String CONTENT_TYPE_APPLICATION_JSON_V1 = "application/json;v=1.0"; public static final String CONTENT_TYPE_APPLICATION_XML_V1 = "application/xml;v=1.0"; public static final String CONTENT_TYPE_APPLICATION_PDF_V1 = "application/pdf;v=1.0"; diff --git a/src/main/java/com/cx/restclient/httpClient/utils/HttpClientHelper.java b/src/main/java/com/cx/restclient/httpClient/utils/HttpClientHelper.java index dcda4b49..e11c07d1 100644 --- a/src/main/java/com/cx/restclient/httpClient/utils/HttpClientHelper.java +++ b/src/main/java/com/cx/restclient/httpClient/utils/HttpClientHelper.java @@ -1,17 +1,18 @@ package com.cx.restclient.httpClient.utils; -import com.cx.restclient.common.ErrorMessage; -import com.cx.restclient.common.ErrorUtil; import com.cx.restclient.exception.CxClientException; import com.cx.restclient.exception.CxHTTPClientException; +import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.JavaType; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.type.TypeFactory; import org.apache.commons.io.IOUtils; import org.apache.http.HttpResponse; +import org.apache.http.entity.StringEntity; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.nio.charset.Charset; import java.util.List; @@ -19,7 +20,21 @@ * Created by Galn on 06/02/2018. */ public abstract class HttpClientHelper { + private HttpClientHelper() { + } + public static T convertToObject(HttpResponse response, Class responseType, boolean isCollection) throws IOException, CxClientException { + + if (responseType != null && responseType.isInstance(response)) { + return (T) response; + } + + // If the caller is asking for the whole response, return the response (instead of just its entity), + // no matter if the entity is empty. + if (responseType != null && responseType.isAssignableFrom(response.getClass())) { + return (T) response; + } + //No content if (responseType == null || response.getEntity() == null || response.getEntity().getContentLength() == 0) { return null; @@ -32,17 +47,21 @@ public static T convertToObject(HttpResponse response, Class responseType if (isCollection) { return convertToCollectionObject(response, TypeFactory.defaultInstance().constructCollectionType(List.class, responseType)); } + //convert to T return convertToStrObject(response, responseType); } private static T convertToStrObject(HttpResponse response, Class valueType) throws CxClientException { - ObjectMapper mapper = new ObjectMapper(); + ObjectMapper mapper = getObjectMapper(); try { if (response.getEntity() == null) { return null; } String json = IOUtils.toString(response.getEntity().getContent(), Charset.defaultCharset()); + if (valueType.equals(String.class)) { + return (T) json; + } return mapper.readValue(json, valueType); } catch (IOException e) { @@ -51,7 +70,7 @@ private static T convertToStrObject(HttpResponse response, Class valueTyp } public static String convertToJson(Object o) throws CxClientException { - ObjectMapper mapper = new ObjectMapper(); + ObjectMapper mapper = getObjectMapper(); try { return mapper.writeValueAsString(o); } catch (Exception e) { @@ -59,8 +78,12 @@ public static String convertToJson(Object o) throws CxClientException { } } + public static StringEntity convertToStringEntity(Object o) throws CxClientException, UnsupportedEncodingException { + return new StringEntity(convertToJson(o)); + } + private static T convertToCollectionObject(HttpResponse response, JavaType javaType) throws CxClientException { - ObjectMapper mapper = new ObjectMapper(); + ObjectMapper mapper = getObjectMapper(); try { String json = IOUtils.toString(response.getEntity().getContent(), Charset.defaultCharset()); return mapper.readValue(json, javaType); @@ -69,16 +92,28 @@ private static T convertToCollectionObject(HttpResponse response, JavaType j } } - public static void validateResponse(HttpResponse response, int status, String message) throws CxClientException { - if (response.getStatusLine().getStatusCode() != status) { - if (ErrorUtil.isServerErrorCodes(response.getStatusLine().getStatusCode())) { - throw new CxClientException(ErrorMessage.SERVICE_UNAVAILABLE.getErrorMessage()); - } + private static ObjectMapper getObjectMapper() { + ObjectMapper result = new ObjectMapper(); + + // Prevent UnrecognizedPropertyException if additional fields are added to API responses. + result.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + return result; + } + + public static void validateResponse(HttpResponse response, int expectedStatus, String message) throws CxClientException { + int actualStatusCode = response.getStatusLine().getStatusCode(); + if (actualStatusCode != expectedStatus) { String responseBody = extractResponseBody(response); - responseBody = responseBody.replace("{", "").replace("}", "").replace(System.getProperty("line.separator"), " ").replace(" ", ""); - throw new CxHTTPClientException(response.getStatusLine().getStatusCode(), message + ": " + responseBody); - } + String readableBody = responseBody.replace("{", "") + .replace("}", "") + .replace(System.getProperty("line.separator"), " ") + .replace(" ", ""); + + String exceptionMessage = String.format("Status code: %d, message: '%s', response body: %s", + actualStatusCode, message, readableBody); + throw new CxHTTPClientException(actualStatusCode, exceptionMessage, responseBody); + } } public static String extractResponseBody(HttpResponse response) { @@ -88,4 +123,5 @@ public static String extractResponseBody(HttpResponse response) { return ""; } } + } diff --git a/src/main/java/com/cx/restclient/osa/dto/CVEReportTableRow.java b/src/main/java/com/cx/restclient/osa/dto/CVEReportTableRow.java index 603c05ad..fb1081a0 100644 --- a/src/main/java/com/cx/restclient/osa/dto/CVEReportTableRow.java +++ b/src/main/java/com/cx/restclient/osa/dto/CVEReportTableRow.java @@ -1,7 +1,13 @@ package com.cx.restclient.osa.dto; +import com.cx.restclient.ast.dto.sca.report.Finding; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + import java.io.Serializable; +import static com.cx.restclient.common.ShragaUtils.formatDate; + +@JsonIgnoreProperties(ignoreUnknown = true) public class CVEReportTableRow implements Serializable { private String name; @@ -18,6 +24,21 @@ public CVEReportTableRow(String name, String severity, String publishDate, Strin this.state = state; } + public CVEReportTableRow(CVE cve) { + this.state = cve.getState().getName(); + this.name = cve.getCveName(); + this.publishDate = cve.getPublishDate(); + this.libraryName = cve.getLibraryId(); + + } + + public CVEReportTableRow(Finding finding){ + this.state = finding.isIgnored()?"NOT_EXPLOITABLE":"EXPLOITABLE"; + this.name = finding.getId(); + this.publishDate = formatDate(finding.getPublishDate(), "yyyy-MM-dd'T'HH:mm:ss", "dd/MM/yy"); + this.libraryName = finding.getPackageId(); + } + public String getName() { return name; } diff --git a/src/main/java/com/cx/restclient/osa/dto/ClientType.java b/src/main/java/com/cx/restclient/osa/dto/ClientType.java new file mode 100644 index 00000000..5a0d8797 --- /dev/null +++ b/src/main/java/com/cx/restclient/osa/dto/ClientType.java @@ -0,0 +1,26 @@ +package com.cx.restclient.osa.dto; + +import lombok.Builder; +import lombok.Getter; +import lombok.Setter; + +@Builder +@Getter +@Setter +public class ClientType { + + public static final ClientType RESOURCE_OWNER = new ClientType("resource_owner_client", + "sast_rest_api cxarm_api", + "014DF517-39D1-4453-B7B3-9930C563627C", + null); + + public static final ClientType CLI = new ClientType("cli_client", + "sast_rest_api offline_access", + "B9D84EA8-E476-4E83-A628-8A342D74D3BD", + null); + + private String clientId; + private String scopes; + private String clientSecret; + private String grantType; +} diff --git a/src/main/java/com/cx/restclient/osa/dto/Content.java b/src/main/java/com/cx/restclient/osa/dto/Content.java index 1dc12eda..c50b4219 100644 --- a/src/main/java/com/cx/restclient/osa/dto/Content.java +++ b/src/main/java/com/cx/restclient/osa/dto/Content.java @@ -1,8 +1,10 @@ package com.cx.restclient.osa.dto; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonRawValue; +@JsonIgnoreProperties(ignoreUnknown = true) public class Content { @JsonRawValue diff --git a/src/main/java/com/cx/restclient/osa/dto/CreateOSAScanRequest.java b/src/main/java/com/cx/restclient/osa/dto/CreateOSAScanRequest.java index c4a1b0a3..f603667b 100644 --- a/src/main/java/com/cx/restclient/osa/dto/CreateOSAScanRequest.java +++ b/src/main/java/com/cx/restclient/osa/dto/CreateOSAScanRequest.java @@ -1,7 +1,9 @@ package com.cx.restclient.osa.dto; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; +@JsonIgnoreProperties(ignoreUnknown = true) public class CreateOSAScanRequest { @JsonProperty("ProjectId") diff --git a/src/main/java/com/cx/restclient/osa/dto/Library.java b/src/main/java/com/cx/restclient/osa/dto/Library.java index 53790e15..e12195dd 100644 --- a/src/main/java/com/cx/restclient/osa/dto/Library.java +++ b/src/main/java/com/cx/restclient/osa/dto/Library.java @@ -1,20 +1,21 @@ package com.cx.restclient.osa.dto; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import java.io.Serializable; -/** - * Created by zoharby on 09/01/2017. - */ @JsonIgnoreProperties(ignoreUnknown = true) public class Library implements Serializable { private String id;//:"36b32b00-9ee6-4e2f-85c9-3f03f26519a9", private String name;//:"lib-name", private String version;//:"lib-version", + @JsonProperty("highUniqueVulnerabilityCount") private int highVulnerabilityCount;//:1, + @JsonProperty("mediumUniqueVulnerabilityCount") private int mediumVulnerabilityCount;//:1, + @JsonProperty("lowUniqueVulnerabilityCount") private int lowVulnerabilityCount;//:1, private String newestVersion;//:"1.0.0", private String newestVersionReleaseDate;//:"2016-12-19T10:16:19.1206743Z", @@ -23,7 +24,7 @@ public class Library implements Serializable { public String getId() { - return id; + return this.id; } public void setId(String id) { @@ -31,7 +32,7 @@ public void setId(String id) { } public String getName() { - return name; + return this.name; } public void setName(String name) { @@ -39,7 +40,7 @@ public void setName(String name) { } public String getVersion() { - return version; + return this.version; } public void setVersion(String version) { @@ -47,7 +48,7 @@ public void setVersion(String version) { } public int getHighVulnerabilityCount() { - return highVulnerabilityCount; + return this.highVulnerabilityCount; } public void setHighVulnerabilityCount(int highVulnerabilityCount) { @@ -55,7 +56,7 @@ public void setHighVulnerabilityCount(int highVulnerabilityCount) { } public int getMediumVulnerabilityCount() { - return mediumVulnerabilityCount; + return this.mediumVulnerabilityCount; } public void setMediumVulnerabilityCount(int mediumVulnerabilityCount) { @@ -63,7 +64,7 @@ public void setMediumVulnerabilityCount(int mediumVulnerabilityCount) { } public int getLowVulnerabilityCount() { - return lowVulnerabilityCount; + return this.lowVulnerabilityCount; } public void setLowVulnerabilityCount(int lowVulnerabilityCount) { @@ -71,7 +72,7 @@ public void setLowVulnerabilityCount(int lowVulnerabilityCount) { } public String getNewestVersion() { - return newestVersion; + return this.newestVersion; } public void setNewestVersion(String newestVersion) { @@ -79,7 +80,7 @@ public void setNewestVersion(String newestVersion) { } public String getNewestVersionReleaseDate() { - return newestVersionReleaseDate; + return this.newestVersionReleaseDate; } public void setNewestVersionReleaseDate(String newestVersionReleaseDate) { @@ -87,7 +88,7 @@ public void setNewestVersionReleaseDate(String newestVersionReleaseDate) { } public int getNumberOfVersionsSinceLastUpdate() { - return numberOfVersionsSinceLastUpdate; + return this.numberOfVersionsSinceLastUpdate; } public void setNumberOfVersionsSinceLastUpdate(int numberOfVersionsSinceLastUpdate) { @@ -95,11 +96,11 @@ public void setNumberOfVersionsSinceLastUpdate(int numberOfVersionsSinceLastUpda } public int getConfidenceLevel() { - return confidenceLevel; + return this.confidenceLevel; } public void setConfidenceLevel(int confidenceLevel) { this.confidenceLevel = confidenceLevel; } -} +} \ No newline at end of file diff --git a/src/main/java/com/cx/restclient/osa/dto/LoginRequest.java b/src/main/java/com/cx/restclient/osa/dto/LoginRequest.java index e0df293b..1807469c 100644 --- a/src/main/java/com/cx/restclient/osa/dto/LoginRequest.java +++ b/src/main/java/com/cx/restclient/osa/dto/LoginRequest.java @@ -1,9 +1,12 @@ package com.cx.restclient.osa.dto; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + /** * Created by: Dorg. * Date: 08/09/2016. */ +@JsonIgnoreProperties(ignoreUnknown = true) public class LoginRequest { private String username; diff --git a/src/main/java/com/cx/restclient/osa/dto/OSAResults.java b/src/main/java/com/cx/restclient/osa/dto/OSAResults.java index 05466be2..039eeea6 100644 --- a/src/main/java/com/cx/restclient/osa/dto/OSAResults.java +++ b/src/main/java/com/cx/restclient/osa/dto/OSAResults.java @@ -1,6 +1,8 @@ package com.cx.restclient.osa.dto; -import com.cx.restclient.cxArm.dto.Violation; +import com.cx.restclient.cxArm.dto.Policy; +import com.cx.restclient.dto.Results; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import java.io.Serializable; import java.util.ArrayList; @@ -9,11 +11,13 @@ import java.util.Map; import static com.cx.restclient.common.ShragaUtils.formatDate; +import static com.cx.restclient.cxArm.utils.CxARMUtils.getPolicyList; /** * Created by Galn on 07/02/2018. */ -public class OSAResults implements Serializable { +@JsonIgnoreProperties(ignoreUnknown = true) +public class OSAResults extends Results implements Serializable { private String osaScanId; private OSASummaryResults results; private List osaLibraries; @@ -28,9 +32,7 @@ public class OSAResults implements Serializable { private String scanStartTime; private String scanEndTime; - private List osaPolicies = new ArrayList<>(); - private List osaViolations = new ArrayList<>(); - + private List osaPolicies = new ArrayList<>(); public OSAResults() { } @@ -182,23 +184,15 @@ public void setScanEndTime(String scanEndTime) { this.scanEndTime = scanEndTime; } - public List getOsaViolations() { - return osaViolations; - } - - public void setOsaViolations(List osaViolations) { - this.osaViolations = osaViolations; - } - - public void addAllViolations(List violations) { - this.osaViolations.addAll(violations); + public void addPolicy(Policy policy) { + this.osaPolicies.addAll(getPolicyList(policy)); } - public List getOsaPolicies() { + public List getOsaPolicies() { return osaPolicies; } - public void setOsaPolicies(List osaPolicies) { + public void setOsaPolicies(List osaPolicies) { this.osaPolicies = osaPolicies; } } diff --git a/src/main/java/com/cx/restclient/osa/dto/ScanConfiguration.java b/src/main/java/com/cx/restclient/osa/dto/ScanConfiguration.java index 2970b248..92b605b7 100644 --- a/src/main/java/com/cx/restclient/osa/dto/ScanConfiguration.java +++ b/src/main/java/com/cx/restclient/osa/dto/ScanConfiguration.java @@ -2,16 +2,17 @@ import com.cx.restclient.sast.dto.Project; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import java.io.File; /** * Created by galn on 21/12/2016. */ +@JsonIgnoreProperties(ignoreUnknown = true) public class ScanConfiguration { private boolean SASTEnabled; - private String OSAEnabled; private String cxOrigin; private String sourceDir; private String tempDir; @@ -60,14 +61,6 @@ public void setSASTEnabled(boolean SASTEnabled) { this.SASTEnabled = SASTEnabled; } - public String getOSAEnabled() { - return OSAEnabled; - } - - public void setOSAEnabled(String OSAEnabled) { - this.OSAEnabled = OSAEnabled; - } - public String getCxOrigin() { return cxOrigin; } diff --git a/src/main/java/com/cx/restclient/osa/utils/OSAUtils.java b/src/main/java/com/cx/restclient/osa/utils/OSAUtils.java index 2e0395b2..0e586b61 100644 --- a/src/main/java/com/cx/restclient/osa/utils/OSAUtils.java +++ b/src/main/java/com/cx/restclient/osa/utils/OSAUtils.java @@ -11,10 +11,7 @@ import java.io.File; import java.nio.charset.Charset; import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.Properties; +import java.util.*; import static com.cx.restclient.common.CxPARAM.CX_REPORT_LOCATION; @@ -24,11 +21,10 @@ public abstract class OSAUtils { private static final String[] SUPPORTED_EXTENSIONS = {"jar", "war", "ear", "aar", "dll", "exe", "msi", "nupkg", "egg", "whl", "tar.gz", "gem", "deb", "udeb", - "dmg", "drpm", "rpm", "pkg.tar.xz", "swf", "swc", "air", "apk", "zip", "gzip", "tar.bz2", "tgz", "c", "cc", "cp", "cpp", "css", "c++", "h", "hh", "hpp", - "hxx", "h++", "m", "mm", "pch", "java", "c#", "cs", "csharp", "go", "goc", "js", "plx", "pm", "ph", "cgi", "fcgi", "psgi", "al", "perl", "t", "p6m", "p6l", "nqp,6pl", "6pm", - "p6", "php", "py", "rb", "swift", "clj", "cljx", "cljs", "cljc"}; + "dmg", "drpm", "rpm", "pkg.tar.xz", "swf", "swc", "air", "apk", "zip", "gzip", "tar.bz2", "tgz", "js"}; private static final String INCLUDE_ALL_EXTENSIONS = "**/**"; + private static final String JSON_EXTENSION = ".json"; public static final String DEFAULT_ARCHIVE_INCLUDES = "**/.*jar,**/*.war,**/*.ear,**/*.sca,**/*.gem,**/*.whl,**/*.egg,**/*.tar,**/*.tar.gz,**/*.tgz,**/*.zip,**/*.rar"; @@ -48,7 +44,7 @@ public static String composeProjectOSASummaryLink(String url, long projectId) { return String.format(url + "/CxWebClient/SPA/#/viewer/project/%s", projectId); } - public static Properties generateOSAScanConfiguration(String folderExclusions, String filterPatterns, String archiveIncludes, String scanFolder, boolean installBeforeScan, Logger log) { + public static Properties generateOSAScanConfiguration(String folderExclusions, String filterPatterns, String archiveIncludes, String sourceDir, boolean installBeforeScan, String osaScanDepth, Logger log) { Properties ret = new Properties(); filterPatterns = StringUtils.defaultString(filterPatterns); archiveIncludes = StringUtils.defaultString(archiveIncludes); @@ -86,24 +82,60 @@ public static Properties generateOSAScanConfiguration(String folderExclusions, S ret.put("archiveIncludes", DEFAULT_ARCHIVE_INCLUDES); } - ret.put("archiveExtractionDepth", "4"); + ret.put("archiveExtractionDepth", StringUtils.isNotEmpty(osaScanDepth) ? osaScanDepth : "4"); + + ret.put("npm.ignoreNpmLsErrors", "true"); if (installBeforeScan) { ret.put("npm.runPreStep", "true"); ret.put("bower.runPreStep", "false"); ret.put("npm.ignoreScripts", "true"); - - ret.put("nuget.resolveDependencies", "true"); - ret.put("nuget.restoreDependencies", "true"); - ret.put("python.resolveDependencies", "true"); - ret.put("python.ignorePipInstallErrors", "true"); + ret.put("php.runPreStep", "true"); + ret.put("sbt.runPreStep", "true"); + setResolveDependencies(ret, "true"); + ret.put("sbt.targetFolder", getSbtTargetFolder(sourceDir)); + } else { + setResolveDependencies(ret, "false"); } - ret.put("d", scanFolder); - + ret.put("d", sourceDir); return ret; } + private static void setResolveDependencies(Properties ret, String resolveDependencies) { + ret.put("gradle.runAssembleCommand", resolveDependencies); + ret.put("nuget.resolveDependencies", resolveDependencies); + ret.put("nuget.restoreDependencies", resolveDependencies); + ret.put("python.resolveDependencies", resolveDependencies); + ret.put("python.ignorePipInstallErrors", resolveDependencies); + ret.put("php.resolveDependencies", resolveDependencies); + ret.put("sbt.resolveDependencies", resolveDependencies); + } + + private static String getSbtTargetFolder(String sourceFolder) { + List files = new ArrayList(); + files = getBuildSbtFiles(sourceFolder, files); + if (!files.isEmpty()) { + return files.get(0).getAbsolutePath().replace("build.sbt", "target"); + } + return "target"; + } + + private static List getBuildSbtFiles(String path, List inputFiles) { + File folder = new File(path); + List files = Arrays.asList(folder.listFiles()); + for (File file : files) { + if (file.isFile()) { + if (file.getName().endsWith("build.sbt")) { + inputFiles.add(file); + } + } else if (file.isDirectory()) { + inputFiles = getBuildSbtFiles(file.getAbsolutePath(), inputFiles); + } + } + return inputFiles; + } + public static void printOSAResultsToConsole(OSAResults osaResults, boolean enableViolations, Logger log) { OSASummaryResults osaSummaryResults = osaResults.getResults(); log.info("----------------------------Checkmarx Scan Results(CxOSA):-------------------------------"); @@ -124,29 +156,59 @@ public static void printOSAResultsToConsole(OSAResults osaResults, boolean enabl log.info("Vulnerable and updated: " + osaSummaryResults.getVulnerableAndUpdated()); log.info("Non-vulnerable libraries: " + osaSummaryResults.getNonVulnerableLibraries()); log.info(""); - if (enableViolations) { - if (osaResults.getOsaPolicies().isEmpty()){ - log.info("Project policy status: compliant"); - }else{ - log.info("Project policy status: violated"); - log.info("OSA violated policies names: " + StringUtils.join(osaResults.getOsaPolicies(), ',')); - } - } log.info("OSA scan results location: " + osaResults.getOsaProjectSummaryLink()); log.info("-----------------------------------------------------------------------------------------"); } - public static void writeJsonToFile(String name, Object jsonObj, File workDirectory, Logger log) { + public static File getWorkDirectory(File filePath, Boolean osaGenerateJsonReport) { + if (filePath == null) { + return null; + } + + if (!osaGenerateJsonReport) { + return filePath; + } + + File workDirectory; + if (!filePath.isAbsolute()) { + workDirectory = new File(System.getProperty("user.dir") + CX_REPORT_LOCATION); + } else { + workDirectory = filePath.getParentFile(); + } + if (!workDirectory.exists()) { + workDirectory.mkdirs(); + } + + return workDirectory; + } + + public static void writeJsonToFile(String name, Object jsonObj, File workDirectory, Boolean cliOsaGenerateJsonReport, Logger log) { try { ObjectMapper objectMapper = new ObjectMapper(); - String now = new SimpleDateFormat("dd_MM_yyyy-HH_mm_ss").format(new Date()); - String fileName = name + "_" + now + ".json"; - File jsonFile = new File(workDirectory + CX_REPORT_LOCATION, fileName); String json = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(jsonObj); - FileUtils.writeStringToFile(jsonFile, json); - log.info(name + " json location: " + workDirectory + CX_REPORT_LOCATION + File.separator + fileName); + + if (cliOsaGenerateJsonReport) { + //workDirectory = new File(workDirectory.getPath().replace(".json", "_" + name + ".json")); + if (!workDirectory.isAbsolute()) { + workDirectory = new File(System.getProperty("user.dir") + CX_REPORT_LOCATION + File.separator + workDirectory); + } + if (!workDirectory.getParentFile().exists()) { + workDirectory.getParentFile().mkdirs(); + } + name = name.endsWith(JSON_EXTENSION) ? name : name + JSON_EXTENSION; + File jsonFile = new File(workDirectory + File.separator + name); + FileUtils.writeStringToFile(jsonFile, json); + log.info(name + " saved under location: " + jsonFile); + } else { + String now = new SimpleDateFormat("dd_MM_yyyy-HH_mm_ss").format(new Date()); + String fileName = name + "_" + now + JSON_EXTENSION; + File jsonFile = new File(workDirectory + CX_REPORT_LOCATION, fileName); + FileUtils.writeStringToFile(jsonFile, json); + log.info(name + " saved under location: " + workDirectory + CX_REPORT_LOCATION + File.separator + fileName); + } } catch (Exception ex) { log.warn("Failed to write OSA JSON report (" + name + ") to file: " + ex.getMessage()); } } + } diff --git a/src/main/java/com/cx/restclient/sast/dto/CreateProjectRequest.java b/src/main/java/com/cx/restclient/sast/dto/CreateProjectRequest.java index 1e611112..bdc0db98 100644 --- a/src/main/java/com/cx/restclient/sast/dto/CreateProjectRequest.java +++ b/src/main/java/com/cx/restclient/sast/dto/CreateProjectRequest.java @@ -38,6 +38,6 @@ public boolean getIsPublic() { } public void setIsPublic(boolean isPublic) { - isPublic = isPublic; + this.isPublic = isPublic; } } diff --git a/src/main/java/com/cx/restclient/sast/dto/CxARMStatus.java b/src/main/java/com/cx/restclient/sast/dto/CxARMStatus.java new file mode 100644 index 00000000..691ee0d3 --- /dev/null +++ b/src/main/java/com/cx/restclient/sast/dto/CxARMStatus.java @@ -0,0 +1,48 @@ +package com.cx.restclient.sast.dto; + + +import com.cx.restclient.dto.BaseStatus; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +/** + * Created by Galn on 07/03/2018. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class CxARMStatus extends BaseStatus { + private CxID project; + private CxID scan; + String status; + String lastSync; + + public CxID getProject() { + return project; + } + + public void setProject(CxID project) { + this.project = project; + } + + public CxID getScan() { + return scan; + } + + public void setScan(CxID scan) { + this.scan = scan; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getLastSync() { + return lastSync; + } + + public void setLastSync(String lastSync) { + this.lastSync = lastSync; + } +} diff --git a/src/main/java/com/cx/restclient/sast/dto/CxARMStatusEnum.java b/src/main/java/com/cx/restclient/sast/dto/CxARMStatusEnum.java new file mode 100644 index 00000000..caf256aa --- /dev/null +++ b/src/main/java/com/cx/restclient/sast/dto/CxARMStatusEnum.java @@ -0,0 +1,25 @@ +package com.cx.restclient.sast.dto; + +/** + * Created by Galn on 07/03/2018. + */ +public enum CxARMStatusEnum { + + IN_PROGRESS("InProgress"), + FINISHED("Finished"), + FAILED("Failed"), + NONE("None"); + + private final String value; + + CxARMStatusEnum(String value) { + this.value = value; + } + + public String value() { + return this.value; + } + + +} + diff --git a/src/main/java/com/cx/restclient/sast/dto/DateAndTime.java b/src/main/java/com/cx/restclient/sast/dto/DateAndTime.java new file mode 100644 index 00000000..5c90cd7a --- /dev/null +++ b/src/main/java/com/cx/restclient/sast/dto/DateAndTime.java @@ -0,0 +1,47 @@ +package com.cx.restclient.sast.dto; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import java.util.Date; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class DateAndTime { + private Date startedOn; + private Date finishedOn; + private Date engineStartedOn; + private Date engineFinishedOn; + + public DateAndTime() { + } + + public Date getStartedOn() { + return startedOn; + } + + public void setStartedOn(Date startedOn) { + this.startedOn = startedOn; + } + + public Date getFinishedOn() { + return finishedOn; + } + + public void setFinishedOn(Date finishedOn) { + this.finishedOn = finishedOn; + } + + public Date getEngineStartedOn() { + return engineStartedOn; + } + + public void setEngineStartedOn(Date engineStartedOn) { + this.engineStartedOn = engineStartedOn; + } + + public Date getEngineFinishedOn() { + return engineFinishedOn; + } + + public void setEngineFinishedOn(Date engineFinishedOn) { + this.engineFinishedOn = engineFinishedOn; + } +} diff --git a/src/main/java/com/cx/restclient/sast/dto/ExcludeSettingsRequest.java b/src/main/java/com/cx/restclient/sast/dto/ExcludeSettingsRequest.java new file mode 100644 index 00000000..1e195a8c --- /dev/null +++ b/src/main/java/com/cx/restclient/sast/dto/ExcludeSettingsRequest.java @@ -0,0 +1,33 @@ +package com.cx.restclient.sast.dto; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +@JsonIgnoreProperties(ignoreUnknown = true) +public class ExcludeSettingsRequest { + private String excludeFoldersPattern; + private String excludeFilesPattern; + + public ExcludeSettingsRequest() { + } + + public ExcludeSettingsRequest(String excludeFoldersPattern, String excludeFilesPattern) { + this.excludeFoldersPattern = excludeFoldersPattern; + this.excludeFilesPattern = excludeFilesPattern; + } + + public String getExcludeFoldersPattern() { + return excludeFoldersPattern; + } + + public void setExcludeFoldersPattern(String excludeFoldersPattern) { + this.excludeFoldersPattern = excludeFoldersPattern; + } + + public String getExcludeFilesPattern() { + return excludeFilesPattern; + } + + public void setExcludeFilesPattern(String excludeFilesPattern) { + this.excludeFilesPattern = excludeFilesPattern; + } +} diff --git a/src/main/java/com/cx/restclient/sast/dto/LastScanResponse.java b/src/main/java/com/cx/restclient/sast/dto/LastScanResponse.java index 6da1944c..eb675f43 100644 --- a/src/main/java/com/cx/restclient/sast/dto/LastScanResponse.java +++ b/src/main/java/com/cx/restclient/sast/dto/LastScanResponse.java @@ -9,6 +9,7 @@ public class LastScanResponse { private long id; private CxNameObj status; + private DateAndTime dateAndTime; public long getId() { return id; @@ -25,4 +26,12 @@ public CxNameObj getStatus() { public void setStatus(CxNameObj status) { this.status = status; } + + public DateAndTime getDateAndTime() { + return dateAndTime; + } + + public void setDateAndTime(DateAndTime dateAndTime) { + this.dateAndTime = dateAndTime; + } } diff --git a/src/main/java/com/cx/restclient/sast/dto/Project.java b/src/main/java/com/cx/restclient/sast/dto/Project.java index 4ba6c835..22528849 100644 --- a/src/main/java/com/cx/restclient/sast/dto/Project.java +++ b/src/main/java/com/cx/restclient/sast/dto/Project.java @@ -8,6 +8,7 @@ @JsonIgnoreProperties(ignoreUnknown = true) public class Project { private long id; + private String owner; private String name; private String teamId; private boolean isPublic; @@ -43,4 +44,12 @@ public boolean getIsPublic() { public void setIsPublic(boolean isPublic) { this.isPublic = isPublic; } + + public String getOwner() { + return owner; + } + + public void setOwner(String owner) { + this.owner = owner; + } } diff --git a/src/main/java/com/cx/restclient/sast/dto/QueueStatus.java b/src/main/java/com/cx/restclient/sast/dto/QueueStatus.java new file mode 100644 index 00000000..e7eaf3b2 --- /dev/null +++ b/src/main/java/com/cx/restclient/sast/dto/QueueStatus.java @@ -0,0 +1,18 @@ +package com.cx.restclient.sast.dto; + +/** + * Created by Galn on 05/02/2018. + */ + +public enum QueueStatus { + New, + PreScan, + SourcePullingAndDeployment, + Queued, + Scanning, + PostScan, + Finished, + Canceled, + Failed; +} + diff --git a/src/main/java/com/cx/restclient/sast/dto/SASTResults.java b/src/main/java/com/cx/restclient/sast/dto/SASTResults.java index 8b9fbdd1..ec3aa652 100644 --- a/src/main/java/com/cx/restclient/sast/dto/SASTResults.java +++ b/src/main/java/com/cx/restclient/sast/dto/SASTResults.java @@ -1,7 +1,7 @@ package com.cx.restclient.sast.dto; import com.cx.restclient.cxArm.dto.Policy; -import com.cx.restclient.cxArm.dto.Violation; +import com.cx.restclient.dto.Results; import java.io.Serializable; import java.text.DateFormat; @@ -9,6 +9,7 @@ import java.text.SimpleDateFormat; import java.util.*; +import static com.cx.restclient.cxArm.utils.CxARMUtils.getPolicyList; import static com.cx.restclient.sast.utils.SASTParam.PROJECT_LINK_FORMAT; import static com.cx.restclient.sast.utils.SASTParam.SCAN_LINK_FORMAT; @@ -16,7 +17,7 @@ /** * Created by Galn on 05/02/2018. */ -public class SASTResults implements Serializable { +public class SASTResults extends Results implements Serializable { private long scanId; @@ -35,10 +36,10 @@ public class SASTResults implements Serializable { private String sastProjectLink; private String sastPDFLink; - private String scanStart; - private String scanTime; - private String scanStartTime; - private String scanEndTime; + private String scanStart = ""; + private String scanTime = ""; + private String scanStartTime = ""; + private String scanEndTime = ""; private String filesScanned; private String LOC; @@ -48,8 +49,7 @@ public class SASTResults implements Serializable { private byte[] PDFReport; private String pdfFileName; - private List sastPolicies = new ArrayList<>(); - private List sastViolations = new ArrayList<>(); + private List sastPolicies = new ArrayList<>(); public enum Severity { @@ -101,6 +101,10 @@ public void setResults(long scanId, SASTStatisticsResponse statisticsResults, St setSastProjectLink(url, projectId); } + public void addPolicy(Policy policy) { + this.sastPolicies.addAll(getPolicyList(policy)); + } + public long getScanId() { return scanId; } @@ -327,14 +331,23 @@ private String formatToDisplayDate(Date date) { return new SimpleDateFormat(displayDatePattern, locale).format(date); } - private Date createStartDate(String scanStart) throws ParseException { - //"Sunday, February 26, 2017 12:17:09 PM" - String oldPattern = "EEEE, MMMM dd, yyyy hh:mm:ss a"; - Locale locale = Locale.ENGLISH; + private Date createStartDate(String scanStart) throws Exception { + DateFormat formatter; + Date formattedDate = null; - DateFormat oldDateFormat = new SimpleDateFormat(oldPattern, locale); + for (SupportedLanguage lang : SupportedLanguage.values()) { + try { + formatter = new SimpleDateFormat(lang.getFormat(), lang.getLocale()); + formattedDate = formatter.parse(scanStart); + break; + } catch (Exception ignored) { + } + } - return oldDateFormat.parse(scanStart); + if(formattedDate == null){ + throw new Exception(String.format("Failed parsing date [%s]", scanStart)); + } + return formattedDate; } private Date createTimeDate(String scanTime) throws ParseException { @@ -348,27 +361,16 @@ private Date createTimeDate(String scanTime) throws ParseException { } private Date createEndDate(Date scanStartDate, Date scanTimeDate) { - long time /*no c*/ = scanStartDate.getTime() + scanTimeDate.getTime(); + long time = scanStartDate.getTime() + scanTimeDate.getTime(); return new Date(time); } - public List getSastViolations() { - return sastViolations; - } - - public void setSastViolations(List sastViolations) { - this.sastViolations = sastViolations; - } - - public void addAllViolations(List violations) { - this.sastViolations.addAll(violations); - } - - public List getSastPolicies() { + public List getSastPolicies() { return sastPolicies; } - public void setSastPolicies(List sastPolicies) { + public void setSastPolicies(List sastPolicies) { this.sastPolicies = sastPolicies; } + } diff --git a/src/main/java/com/cx/restclient/sast/dto/ScanWithSettingsResponse.java b/src/main/java/com/cx/restclient/sast/dto/ScanWithSettingsResponse.java new file mode 100644 index 00000000..a4babfbb --- /dev/null +++ b/src/main/java/com/cx/restclient/sast/dto/ScanWithSettingsResponse.java @@ -0,0 +1,20 @@ +package com.cx.restclient.sast.dto; + +public class ScanWithSettingsResponse { + private int id; + + public ScanWithSettingsResponse(int id) { + this.id = id; + } + + public ScanWithSettingsResponse() { + } + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } +} diff --git a/src/main/java/com/cx/restclient/sast/dto/SupportedLanguage.java b/src/main/java/com/cx/restclient/sast/dto/SupportedLanguage.java new file mode 100644 index 00000000..0d8e70d2 --- /dev/null +++ b/src/main/java/com/cx/restclient/sast/dto/SupportedLanguage.java @@ -0,0 +1,36 @@ +package com.cx.restclient.sast.dto; + +import java.util.Locale; + +public enum SupportedLanguage { + + ENGLISH_US(Locale.US, "EEEE, MMMM dd, yyyy hh:mm:ss a"), + ENGLISH_GB(Locale.ENGLISH, "dd MMMM yyyy hh:mm"), + FRENCH_FR(Locale.FRENCH, "EEEE dd MMMM yyyy hh:mm"), + PORTUGUESE_PT(new Locale("pt"), "dd 'de' MMMM 'de' yyyy hh:mm"), + SPANISH(new Locale("es"), "EEEE, dd 'de' MMMM 'de' YYYY hh:mm"), + RUSSIAN(new Locale("ru"), "dd MMMM yyyy 'г.' hh:mm"); + + //TODO: Add fitting format +// JAPANESE(new Locale("ja-JP"), "ss"), +// KOREAN(new Locale("ko-KR"), "ss"), +// PORTUGUESE_BR(new Locale("pt-BR"), "ss"), +// CHINESE_CN(new Locale("zn-CN"), "ss"), +// CHINESE_TW(new Locale("zn-TW"), "ss"); + + private final Locale locale; + private final String format; + + SupportedLanguage(Locale locale, String format) { + this.format = format; + this.locale = locale; + } + + public Locale getLocale() { + return locale; + } + + public String getFormat() { + return format; + } +} diff --git a/src/main/java/com/cx/restclient/sast/dto/UpdateScanStatusRequest.java b/src/main/java/com/cx/restclient/sast/dto/UpdateScanStatusRequest.java index 3f27b915..df54beef 100644 --- a/src/main/java/com/cx/restclient/sast/dto/UpdateScanStatusRequest.java +++ b/src/main/java/com/cx/restclient/sast/dto/UpdateScanStatusRequest.java @@ -1,8 +1,11 @@ package com.cx.restclient.sast.dto; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + /** * Created by Galn on 18/03/2018. */ +@JsonIgnoreProperties(ignoreUnknown = true) public class UpdateScanStatusRequest { private String status; diff --git a/src/main/java/com/cx/restclient/sast/utils/LegacyClient.java b/src/main/java/com/cx/restclient/sast/utils/LegacyClient.java new file mode 100644 index 00000000..a2732196 --- /dev/null +++ b/src/main/java/com/cx/restclient/sast/utils/LegacyClient.java @@ -0,0 +1,448 @@ +package com.cx.restclient.sast.utils; + + +import com.cx.restclient.common.UrlUtils; +import com.cx.restclient.configuration.CxScanConfig; +import com.cx.restclient.cxArm.dto.CxArmConfig; +import com.cx.restclient.dto.*; +import com.cx.restclient.exception.CxClientException; +import com.cx.restclient.exception.CxHTTPClientException; +import com.cx.restclient.httpClient.CxHttpClient; +import com.cx.restclient.osa.dto.ClientType; +import com.cx.restclient.sast.dto.CreateProjectRequest; +import com.cx.restclient.sast.dto.CxNameObj; +import com.cx.restclient.sast.dto.Preset; +import com.cx.restclient.sast.dto.Project; +import org.apache.commons.lang.StringUtils; +import org.apache.http.client.HttpResponseException; +import org.apache.http.entity.StringEntity; +import org.slf4j.Logger; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.List; + +import static com.cx.restclient.common.CxPARAM.*; +import static com.cx.restclient.httpClient.utils.ContentType.CONTENT_TYPE_APPLICATION_JSON_V1; +import static com.cx.restclient.httpClient.utils.HttpClientHelper.convertToJson; +import static com.cx.restclient.sast.utils.SASTParam.*; + +/** + * Common parent for SAST and OSA clients. + * Extracted from {@link com.cx.restclient.CxClientDelegator} for better maintainability. + */ +public abstract class LegacyClient { + + private static final String DEFAULT_AUTH_API_PATH = "CxRestApi/auth/" + AUTHENTICATION; + protected CxHttpClient httpClient; + protected CxScanConfig config; + protected Logger log; + private String teamPath; + protected long projectId; + private State state = State.SUCCESS; + private boolean isNewProject = false; + + public LegacyClient(CxScanConfig config, Logger log) throws MalformedURLException { + this.config = config; + this.log = log; + initHttpClient(config, log); + validateConfig(config); + } + + public void setConfig(CxScanConfig config) { + this.config = config; + } + + public void close() { + if (httpClient != null) { + httpClient.close(); + } + } + + public boolean isIsNewProject() { + return isNewProject; + } + + public void setIsNewProject(boolean isNewProject) { + this.isNewProject = isNewProject; + } + + public long resolveProjectId() throws IOException { + List projects = getProjectByName(config.getProjectName(), config.getTeamId(), teamPath); + + if (projects == null || projects.isEmpty()) { // Project is new + if (config.getDenyProject()) { + throw new CxClientException(DENY_NEW_PROJECT_ERROR.replace("{projectName}", config.getProjectName())); + } + //Create newProject + CreateProjectRequest request = new CreateProjectRequest(config.getProjectName(), config.getTeamId(), config.getPublic()); + log.info("Project not found, creating a new one.: '{}' with Team '{}'", config.getProjectName(), teamPath); + projectId = createNewProject(request, teamPath).getId(); + log.info("Created a project with ID {}", projectId); + setIsNewProject(true); + } else { + projectId = projects.get(0).getId(); + setIsNewProject(false); + log.info("Project already exists with ID {}", projectId); + } + + return projectId; + } + + + public String configureTeamPath() throws IOException, CxClientException { + if (StringUtils.isEmpty(config.getTeamPath())) { + List teamList = populateTeamList(); + // If there is no chosen teamPath, just add first one from the teams + // list as default + if (StringUtils.isEmpty(teamPath) && teamList != null && !teamList.isEmpty()) { + teamPath = teamList.get(0).getFullName(); + } + } else { + teamPath = config.getTeamPath(); + } + httpClient.setTeamPathHeader(teamPath); + log.debug(String.format(" setTeamPathHeader %s", teamPath)); + return teamPath; + } + + public List getTeamList() throws IOException, CxClientException { + + return populateTeamList(); + } + + private List populateTeamList() throws IOException { + return (List) httpClient.getRequest(CXTEAMS, CONTENT_TYPE_APPLICATION_JSON_V1, Team.class, 200, "team list", true); + } + + + public String getToken() throws IOException, CxClientException { + LoginSettings settings = getDefaultLoginSettings(); + settings.setClientTypeForPasswordAuth(ClientType.CLI); + final TokenLoginResponse tokenLoginResponse = getHttpClient().generateToken(settings); + return tokenLoginResponse.getRefresh_token(); + } + + public void revokeToken(String token) throws IOException, CxClientException { + getHttpClient().revokeToken(token); + } + + + private Project createNewProject(CreateProjectRequest request, String teamPath) throws IOException { + String json = convertToJson(request); + httpClient.setTeamPathHeader(teamPath); + StringEntity entity = new StringEntity(json, StandardCharsets.UTF_8); + return httpClient.postRequest(CREATE_PROJECT, CONTENT_TYPE_APPLICATION_JSON_V1, entity, Project.class, 201, "create new project: " + request.getName()); + } + + private List getProjectByName(String projectName, String teamId, String teamPath) throws IOException, CxClientException { + projectName = URLEncoder.encode(projectName, "UTF-8"); + String projectNamePath = SAST_GET_PROJECT.replace("{name}", projectName).replace("{teamId}", teamId); + List projects = null; + try { + httpClient.setTeamPathHeader(teamPath); + projects = (List) httpClient.getRequest(projectNamePath, CONTENT_TYPE_APPLICATION_JSON_V1, Project.class, 200, "project by name: " + projectName, true); + } catch (CxHTTPClientException ex) { + if (ex.getStatusCode() != 404) { + throw ex; + } + } + return projects; + } + + private void initHttpClient(CxScanConfig config, Logger log) throws MalformedURLException { + + if (!org.apache.commons.lang3.StringUtils.isEmpty(config.getUrl())) { + httpClient = new CxHttpClient( + UrlUtils.parseURLToString(config.getUrl(), "CxRestAPI/"), + config.getCxOrigin(), + config.getCxOriginUrl(), + config.isDisableCertificateValidation(), + config.isUseSSOLogin(), + config.getRefreshToken(), + config.isProxy(), + config.getProxyConfig(), + log, + config.getNTLM()); + } + } + + public void initiate() throws CxClientException { + try { + if (config.isSastOrOSAEnabled()) { + String version = getCxVersion(); + login(version); + resolveTeam(); + //httpClient.setTeamPathHeader(this.teamPath); + if (config.isSastEnabled()) { + resolvePreset(); + } + if (config.getEnablePolicyViolations()) { + resolveCxARMUrl(); + } + resolveEngineConfiguration(); + resolveProjectId(); + } + } catch (Exception e) { + throw new CxClientException(e); + } + } + + public String getCxVersion() throws IOException, CxClientException { + String version; + try { + config.setCxVersion(httpClient.getRequest(CX_VERSION, CONTENT_TYPE_APPLICATION_JSON_V1, CxVersion.class, 200, "cx Version", false)); + String hotfix = ""; + try { + if (config.getCxVersion().getHotFix() != null && Integer.parseInt(config.getCxVersion().getHotFix()) > 0) { + hotfix = " Hotfix [" + config.getCxVersion().getHotFix() + "]."; + } + } catch (Exception ex) { + } + + version = config.getCxVersion().getVersion(); + log.info("Checkmarx server version [" + config.getCxVersion().getVersion() + "]." + hotfix); + + } catch (Exception ex) { + version = "lower than 9.0"; + log.debug("Checkmarx server version [lower than 9.0]"); + } + return version; + } + + public void login() throws IOException { + String version = getCxVersion(); + login(version); + } + + public void login(String version) throws IOException, CxClientException { + // perform login to server + log.info("Logging into the Checkmarx service."); + + if (config.getToken() != null) { + httpClient.setToken(config.getToken()); + return; + } + LoginSettings settings = getDefaultLoginSettings(); + settings.setRefreshToken(config.getRefreshToken()); + settings.setVersion(version); + httpClient.login(settings); + } + + public LoginSettings getDefaultLoginSettings() throws MalformedURLException { + String baseUrl = UrlUtils.parseURLToString(config.getUrl(), DEFAULT_AUTH_API_PATH); + LoginSettings result = LoginSettings.builder() + .accessControlBaseUrl(baseUrl) + .username(config.getUsername()) + .password(config.getPassword()) + .clientTypeForPasswordAuth(ClientType.RESOURCE_OWNER) + .clientTypeForRefreshToken(ClientType.CLI) + .build(); + + result.getSessionCookies().addAll(config.getSessionCookie()); + + return result; + } + + + public CxHttpClient getHttpClient() { + return httpClient; + } + + private void resolveEngineConfiguration() throws IOException { + if (config.getEngineConfigurationId() == null && config.getEngineConfigurationName() == null) { + config.setEngineConfigurationId(1); + } else if (config.getEngineConfigurationName() != null) { + final List engineConfigurations = getEngineConfiguration(); + for (EngineConfiguration engineConfiguration : engineConfigurations) { + if (engineConfiguration.getName().equalsIgnoreCase(config.getEngineConfigurationName())) { + config.setEngineConfigurationId(engineConfiguration.getId()); + log.info(String.format("Engine configuration: \"%s\" was validated in server", config.getEngineConfigurationName())); + } + } + if (config.getEngineConfigurationId() == null) { + throw new CxClientException("Engine configuration: \"" + config.getEngineConfigurationName() + "\" was not found in server"); + } + } + } + + public List getEngineConfiguration() throws IOException { + configureTeamPath(); + httpClient.setTeamPathHeader(this.teamPath); + return (List) httpClient.getRequest(SAST_ENGINE_CONFIG, CONTENT_TYPE_APPLICATION_JSON_V1, EngineConfiguration.class, 200, "engine configurations", true); + } + + + public void validateConfig(CxScanConfig config) throws CxClientException { + String message = null; + if (config == null) { + message = "Non-null config must be provided."; + } else if (org.apache.commons.lang3.StringUtils.isEmpty(config.getUrl()) && config.isSastOrOSAEnabled()) { + message = "Server URL is required when SAST or OSA is enabled."; + } + if (message != null) { + throw new CxClientException(message); + } + } + + private void resolveTeam() throws CxClientException, IOException { + + configureTeamPath(); + + if (config.getTeamId() == null) { + config.setTeamId(getTeamIdByName(config.getTeamPath())); + } + + printTeamPath(); + } + + public String getTeamIdByName(String teamName) throws CxClientException, IOException { + teamName = replaceDelimiters(teamName); + List allTeams = getTeamList(); + for (Team team : allTeams) { + String fullName = replaceDelimiters(team.getFullName()); + if (fullName.equalsIgnoreCase(teamName)) { //TODO caseSenesitive + return team.getId(); + } + } + throw new CxClientException("Could not resolve team ID from team name: " + teamName); + } + + private String replaceDelimiters(String teamName) { + while (teamName.contains("\\") || teamName.contains("//")) { + teamName = teamName.replace("\\", "/"); + teamName = teamName.replace("//", "/"); + } + return teamName; + } + + private CxArmConfig getCxARMConfig() throws IOException, CxClientException { + httpClient.setTeamPathHeader(this.teamPath); + return httpClient.getRequest(CX_ARM_URL, CONTENT_TYPE_APPLICATION_JSON_V1, CxArmConfig.class, 200, "CxARM URL", false); + } + + private void resolveCxARMUrl() throws CxClientException { + try { + this.config.setCxARMUrl(getCxARMConfig().getCxARMPolicyURL()); + } catch (Exception ex) { + throw new CxClientException("CxARM is not available. Policy violations cannot be calculated: " + ex.getMessage()); + } + } + + private void resolvePreset() throws CxClientException, IOException { + if (config.getPresetId() == null) { + config.setPresetId(getPresetIdByName(config.getPresetName())); + } + printPresetName(); + } + + public int getPresetIdByName(String presetName) throws CxClientException, IOException { + List allPresets = getPresetList(); + for (Preset preset : allPresets) { + if (preset.getName().equalsIgnoreCase(presetName)) { //TODO caseSenesitive- checkkk + return preset.getId(); + } + } + + throw new CxClientException("Could not resolve preset ID from preset name: " + presetName); + } + + public List getPresetList() throws IOException, CxClientException { + configureTeamPath(); + return (List) httpClient.getRequest(CXPRESETS, CONTENT_TYPE_APPLICATION_JSON_V1, Preset.class, 200, "preset list", true); + } + + + private void printPresetName() { + try { + String presetName = config.getPresetName(); + if (presetName == null) { + presetName = getPresetById(config.getPresetId()).getName(); + } + log.info(String.format("preset name: %s", presetName)); + } catch (Exception e) { + log.warn("Error getting preset name."); + } + } + + public Preset getPresetById(int presetId) throws IOException, CxClientException { + httpClient.setTeamPathHeader(this.teamPath); + return httpClient.getRequest(CXPRESETS + "/" + presetId, CONTENT_TYPE_APPLICATION_JSON_V1, Preset.class, 200, "preset by id", false); + } + + private void printTeamPath() { + try { + this.teamPath = config.getTeamPath(); + if (this.teamPath == null) { + this.teamPath = getTeamNameById(config.getTeamId()); + } + log.info(String.format("full team path: %s", this.teamPath)); + } catch (Exception e) { + log.warn("Error getting team path."); + } + } + + + public String getTeamNameById(String teamId) throws CxClientException, IOException { + List allTeams = getTeamList(); + for (Team team : allTeams) { + if (teamId.equals(team.getId())) { + return team.getFullName(); + } + } + throw new CxClientException("Could not resolve team name from id: " + teamId); + } + + + public List getAllProjects() throws IOException, CxClientException { + List projects = null; + configureTeamPath(); + + try { + projects = (List) httpClient.getRequest(SAST_GET_ALL_PROJECTS, CONTENT_TYPE_APPLICATION_JSON_V1, Project.class, 200, "all projects", true); + } catch (HttpResponseException ex) { + if (ex.getStatusCode() != 404) { + throw ex; + } + } + return projects; + } + + public Project getProjectById(String projectId, String contentType) throws IOException, CxClientException { + String projectNamePath = SAST_GET_PROJECT_BY_ID.replace("{projectId}", projectId); + Project projects = null; + try { + httpClient.setTeamPathHeader(this.teamPath); + projects = httpClient.getRequest(projectNamePath, contentType, Project.class, 200, "project by id: " + projectId, false); + } catch (CxHTTPClientException ex) { + if (ex.getStatusCode() != 404) { + throw ex; + } + } + return projects; + } + + + public List getConfigurationSetList() throws IOException, CxClientException { + configureTeamPath(); + return (List) httpClient.getRequest(SAST_ENGINE_CONFIG, CONTENT_TYPE_APPLICATION_JSON_V1, CxNameObj.class, 200, "engine configurations", true); + } + + public String getTeamPath() { + return teamPath; + } + + public void setTeamPath(String teamPath) { + this.teamPath = teamPath; + } + + public State getState() { + return state; + } + + public void setState(State state) { + this.state = state; + } +} diff --git a/src/main/java/com/cx/restclient/sast/utils/SASTParam.java b/src/main/java/com/cx/restclient/sast/utils/SASTParam.java index 0ca54d5d..193d1f7e 100644 --- a/src/main/java/com/cx/restclient/sast/utils/SASTParam.java +++ b/src/main/java/com/cx/restclient/sast/utils/SASTParam.java @@ -11,10 +11,17 @@ public class SASTParam { public static final String SAST_CREATE_SCAN = "sast/scans"; //Run a new Scan public static final String SAST_SCAN = "sast/scans/{scanId}"; //Get Scan status (by scan ID) public static final String SAST_QUEUE_SCAN_STATUS = "sast/scansQueue/{scanId}"; + public static final String SAST_GET_PROJECT_BY_ID = "projects/{projectId}"; public static final String SAST_GET_PROJECT = "projects?projectname={name}&teamid={teamId}";// Get project) - public static final String SAST_GET_All_PROJECTS = "projects";// Get project) + public static final String SAST_GET_ALL_PROJECTS = "projects";// Get project) public static final String SAST_ZIP_ATTACHMENTS = "projects/{projectId}/sourceCode/attachments";//Attach ZIP file public static final String SAST_GET_PROJECT_SCANS = "sast/scans?projectId={projectId}"; + public static final String SAST_GET_QUEUED_SCANS = "sast/scansQueue?projectId={projectId}"; + + + public static final String SAST_CREATE_REMOTE_SOURCE_SCAN = "projects/%s/sourceCode/remoteSettings/%s/%s"; + public static final String SAST_EXCLUDE_FOLDERS_FILES_PATTERNS = "projects/%s/sourceCode/excludeSettings"; + //Once it has results @@ -22,6 +29,8 @@ public class SASTParam { public static final String SAST_CREATE_REPORT = "reports/sastScan/"; //Create new report (get ID) public static final String SAST_GET_REPORT_STATUS = "reports/sastScan/{reportId}/status"; //Get report status public static final String SAST_GET_REPORT = "reports/sastScan/{reportId}"; //Get report status + public static final String SAST_GET_CXARM_STATUS = "sast/projects/{projectId}/publisher/policyFindings/status"; //Get report status + //ZIP PARAMS public static final long MAX_ZIP_SIZE_BYTES = 2147483648L; diff --git a/src/main/java/com/cx/restclient/sast/utils/SASTUtils.java b/src/main/java/com/cx/restclient/sast/utils/SASTUtils.java index 403541d8..1870ba81 100644 --- a/src/main/java/com/cx/restclient/sast/utils/SASTUtils.java +++ b/src/main/java/com/cx/restclient/sast/utils/SASTUtils.java @@ -1,55 +1,46 @@ package com.cx.restclient.sast.utils; -import com.cx.restclient.exception.CxClientException; -import com.cx.restclient.sast.dto.CxXMLResults; -import com.cx.restclient.sast.dto.SASTResults; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; +import static com.cx.restclient.common.CxPARAM.CX_REPORT_LOCATION; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.util.Collections; import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; import javax.xml.bind.Unmarshaller; -import java.io.ByteArrayInputStream; -import java.io.File; -import java.text.SimpleDateFormat; -import java.util.Date; -import static com.cx.restclient.common.CxPARAM.CX_REPORT_LOCATION; -import static com.cx.restclient.sast.utils.SASTParam.PDF_REPORT_NAME; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; + +import com.cx.restclient.exception.CxClientException; +import com.cx.restclient.sast.dto.CxXMLResults; +import com.cx.restclient.sast.dto.SASTResults; +import com.sun.xml.bind.v2.ContextFactory; /** * Created by Galn on 07/02/2018. */ public abstract class SASTUtils { - public static void deleteTempZipFile(File zipTempFile, Logger log) { - if (zipTempFile.exists() && !zipTempFile.delete()) { - log.warn("Failed to delete temporary zip file: " + zipTempFile.getAbsolutePath()); - } else { - log.info("Temporary file deleted"); - } - } - public static CxXMLResults convertToXMLResult(byte[] cxReport) throws CxClientException { CxXMLResults reportObj = null; - ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(cxReport); - try { - JAXBContext jaxbContext = JAXBContext.newInstance(CxXMLResults.class); + try (ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(cxReport)) { + + JAXBContext jaxbContext = ContextFactory.createContext(CxXMLResults.class.getPackage().getName(), + CxXMLResults.class.getClassLoader(), Collections.emptyMap()); Unmarshaller unmarshaller = jaxbContext.createUnmarshaller(); reportObj = (CxXMLResults) unmarshaller.unmarshal(byteArrayInputStream); - } catch (JAXBException e) { + } catch (JAXBException | IOException e) { throw new CxClientException("Failed to parse xml report: " + e.getMessage(), e); - } finally { - IOUtils.closeQuietly(byteArrayInputStream); } return reportObj; } - public static void printSASTResultsToConsole(SASTResults sastResults,boolean enableViolations, Logger log) { + public static void printSASTResultsToConsole(SASTResults sastResults, boolean enableViolations, Logger log) { String highNew = sastResults.getNewHigh() > 0 ? " (" + sastResults.getNewHigh() + " new)" : ""; String mediumNew = sastResults.getNewMedium() > 0 ? " (" + sastResults.getNewMedium() + " new)" : ""; @@ -62,23 +53,38 @@ public static void printSASTResultsToConsole(SASTResults sastResults,boolean ena log.info("Low severity results: " + sastResults.getLow() + lowNew); log.info("Information severity results: " + sastResults.getInformation() + infoNew); log.info(""); - /* if (enableViolations && !sastResults.getSastPolicies().isEmpty()) { - log.info("SAST violated policies names: " + StringUtils.join(sastResults.getSastPolicies(), ',')); - }*/ log.info("Scan results location: " + sastResults.getSastScanLink()); log.info("------------------------------------------------------------------------------------------\n"); } //PDF Report - public static String writePDFReport(byte[] scanReport, File workspace, Logger log) { - String now = new SimpleDateFormat("dd_MM_yyyy-HH_mm_ss").format(new Date()); - String pdfFileName = PDF_REPORT_NAME + "_" + now + ".pdf"; + public static String writePDFReport(byte[] scanReport, File workspace, String pdfFileName, Logger log) { try { FileUtils.writeByteArrayToFile(new File(workspace + CX_REPORT_LOCATION, pdfFileName), scanReport); log.info("PDF report location: " + workspace + CX_REPORT_LOCATION + File.separator + pdfFileName); } catch (Exception e) { log.error("Failed to write PDF report to workspace: ", e.getMessage()); + pdfFileName = ""; } return pdfFileName; } + + // CLI Report/s + public static void writeReport(byte[] scanReport, String reportName, Logger log) { + try { + File reportFile = new File(reportName); + if (!reportFile.isAbsolute()) { + reportFile = new File(System.getProperty("user.dir") + CX_REPORT_LOCATION + File.separator + reportFile); + } + + if (!reportFile.getParentFile().exists()) { + reportFile.getParentFile().mkdirs(); + } + + FileUtils.writeByteArrayToFile(reportFile, scanReport); + log.info("report location: " + reportFile.getAbsolutePath()); + } catch (Exception e) { + log.error("Failed to write report: ", e.getMessage()); + } + } } diff --git a/src/main/java/com/cx/restclient/sast/utils/State.java b/src/main/java/com/cx/restclient/sast/utils/State.java new file mode 100644 index 00000000..0a2d146e --- /dev/null +++ b/src/main/java/com/cx/restclient/sast/utils/State.java @@ -0,0 +1,6 @@ +package com.cx.restclient.sast.utils; + +public enum State { + SUCCESS, + FAILED +} diff --git a/src/main/java/com/cx/restclient/sast/utils/zip/CxZip.java b/src/main/java/com/cx/restclient/sast/utils/zip/CxZip.java index aed2a75e..d8a051ac 100644 --- a/src/main/java/com/cx/restclient/sast/utils/zip/CxZip.java +++ b/src/main/java/com/cx/restclient/sast/utils/zip/CxZip.java @@ -1,13 +1,13 @@ package com.cx.restclient.sast.utils.zip; +import com.cx.restclient.dto.PathFilter; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; +import java.io.ByteArrayOutputStream; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.OutputStream; public class CxZip { @@ -23,34 +23,32 @@ public CxZip(String tempFileName, long maxZipSizeInBytes, Logger log) { this.maxZipSizeInBytes = maxZipSizeInBytes; } - public File zipWorkspaceFolder(File baseDir, String[] includes, String[] excludes) - throws IOException { - log.info("Zipping workspace: '" + baseDir + "'"); + public byte[] zipWorkspaceFolder(File baseDir, PathFilter filter) throws IOException { + log.debug("Zipping workspace: '" + baseDir + "'"); ZipListener zipListener = new ZipListener() { public void updateProgress(String fileName, long size) { numOfZippedFiles++; - log.info("Zipping (" + FileUtils.byteCountToDisplaySize(size) + "): " + fileName); + log.debug("Zipping (" + FileUtils.byteCountToDisplaySize(size) + "): " + fileName); } }; - File tempFile = File.createTempFile(tempFileName, ".bin"); - OutputStream fileOutputStream = new FileOutputStream(tempFile); - - try { - new Zipper(log).zip(baseDir, includes, excludes, fileOutputStream, maxZipSizeInBytes, zipListener); - } catch (Zipper.MaxZipSizeReached e) { - tempFile.delete(); - throw new IOException("Reached maximum upload size limit of " + FileUtils.byteCountToDisplaySize(maxZipSizeInBytes)); - } catch (Zipper.NoFilesToZip e) { - throw new IOException("No files to zip"); - } + byte[] zipFileBA; + try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) { + try { + new Zipper(log).zip(baseDir, filter.getIncludes(), filter.getExcludes(), byteArrayOutputStream, maxZipSizeInBytes, zipListener); + } catch (Zipper.MaxZipSizeReached e) { + throw new IOException("Reached maximum upload size limit of " + FileUtils.byteCountToDisplaySize(maxZipSizeInBytes)); + } catch (Zipper.NoFilesToZip e) { + throw new IOException("No files to zip"); + } - log.info("Zipping complete with " + numOfZippedFiles + " files, total compressed size: " + - FileUtils.byteCountToDisplaySize(tempFile.length())); - log.info("Temporary file with zipped sources was created at: '" + tempFile.getAbsolutePath() + "'"); + log.debug("Zipping complete with " + numOfZippedFiles + " files, total compressed size: " + + FileUtils.byteCountToDisplaySize(byteArrayOutputStream.size())); - return tempFile; + zipFileBA = byteArrayOutputStream.toByteArray(); + } + return zipFileBA; } public CxZip setMaxZipSizeInBytes(long maxZipSizeInBytes) { diff --git a/src/main/java/com/cx/restclient/sast/utils/zip/CxZipUtils.java b/src/main/java/com/cx/restclient/sast/utils/zip/CxZipUtils.java index d47d9e9e..9e1a2c54 100644 --- a/src/main/java/com/cx/restclient/sast/utils/zip/CxZipUtils.java +++ b/src/main/java/com/cx/restclient/sast/utils/zip/CxZipUtils.java @@ -1,34 +1,35 @@ package com.cx.restclient.sast.utils.zip; -import com.cx.restclient.common.ShragaUtils; import com.cx.restclient.configuration.CxScanConfig; +import com.cx.restclient.dto.PathFilter; +import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import java.io.File; import java.io.IOException; -import java.util.List; -import java.util.Map; +import static com.cx.restclient.sast.utils.SASTParam.MAX_ZIP_SIZE_BYTES; import static com.cx.restclient.sast.utils.SASTParam.TEMP_FILE_NAME_TO_ZIP; /** * CxZipUtils generates the patterns used for zipping the workspace folder */ - - public abstract class CxZipUtils { - public static File zipWorkspaceFolder(CxScanConfig config, long maxZipBytes, Logger log) throws IOException { - Map> stringListMap = ShragaUtils.generateIncludesExcludesPatternLists(config.getSastFolderExclusions(), config.getSastFilterPattern(), log); - List includes = stringListMap.get(ShragaUtils.INCLUDES_LIST); - List excludes = stringListMap.get(ShragaUtils.EXCLUDES_LIST); - - CxZip cxZip = new CxZip(TEMP_FILE_NAME_TO_ZIP, maxZipBytes, log); - - return cxZip.zipWorkspaceFolder(new File(config.getSourceDir()), includes.toArray(new String[includes.size()]), excludes.toArray(new String[excludes.size()])); - + public synchronized static byte[] getZippedSources(CxScanConfig config, PathFilter filter, String sourceDir, Logger log) throws IOException { + byte[] zipFile = config.getZipFile() != null ? FileUtils.readFileToByteArray(config.getZipFile()) : null; + if (zipFile == null) { + log.debug("----------------------------------- Start zipping files :------------------------------------"); + Long maxZipSize = config.getMaxZipSize() != null ? config.getMaxZipSize() * 1024 * 1024 : MAX_ZIP_SIZE_BYTES; + + CxZip cxZip = new CxZip(TEMP_FILE_NAME_TO_ZIP, maxZipSize, log); + zipFile = cxZip.zipWorkspaceFolder(new File(sourceDir), filter); + log.debug("----------------------------------- Finish zipping files :------------------------------------"); + } + return zipFile; } + } diff --git a/src/main/java/com/cx/restclient/sast/utils/zip/NewCxZipFile.java b/src/main/java/com/cx/restclient/sast/utils/zip/NewCxZipFile.java new file mode 100644 index 00000000..2d5fa5d1 --- /dev/null +++ b/src/main/java/com/cx/restclient/sast/utils/zip/NewCxZipFile.java @@ -0,0 +1,142 @@ +package com.cx.restclient.sast.utils.zip; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.tools.ant.DirectoryScanner; +import org.apache.tools.zip.ZipEntry; +import org.apache.tools.zip.ZipOutputStream; +import org.slf4j.Logger; + +import java.io.*; +import java.util.List; + + +public class NewCxZipFile implements Closeable { + + private static final double AVERAGE_ZIP_COMPRESSION_RATIO = 4.0D; + + private final Logger log; + private final long maxSize; + private final ZipListener listener; + private final OutputStream outputStream; + private final ZipOutputStream zipOutputStream; + private long fileCount; + private long compressedSize; + + public NewCxZipFile(File zipFile, long maxZipSizeInBytes, Logger log) throws IOException { + this.log = log; + this.maxSize = maxZipSizeInBytes; + this.fileCount = 0; + this.compressedSize = 0; + this.listener = (fileName, size) -> { + fileCount++; + if (log.isInfoEnabled()) + log.info("Zipping ( {} ): {}", FileUtils.byteCountToDisplaySize(size), fileName); + }; + outputStream = new FileOutputStream(zipFile); + zipOutputStream = new ZipOutputStream(outputStream); + zipOutputStream.setEncoding("UTF8"); + } + + + public void zipContentAsFile(String pathInZip, byte[] content) throws IOException { + + validateNextFileWillNotReachMaxCompressedSize((double) content.length); + + try (ByteArrayInputStream inputStream = new ByteArrayInputStream(content)) { + compressedSize += InsertZipEntry(zipOutputStream, pathInZip, inputStream); + if (listener != null) { + listener.updateProgress(pathInZip, compressedSize); + } + } catch (IOException ioException) { + log.warn(String.format("Failed to add file to archive: %s", pathInZip), ioException); + } + } + + public void close() { + IOUtils.closeQuietly(zipOutputStream); + IOUtils.closeQuietly(outputStream); + } + + public void addMultipleFilesToArchive(File baseDir, List relativePaths) throws IOException { + assert baseDir != null : "baseDir must not be null"; + assert outputStream != null : "outputStream must not be null"; + + int len$ = relativePaths.size(); + + for (int i$ = 0; i$ < len$; ++i$) { + String fileName = relativePaths.get(i$); + + File file = new File(baseDir, fileName); + if (!file.canRead()) { + log.warn("Skipping unreadable file: {}", file); + continue; + } + validateNextFileWillNotReachMaxCompressedSize((double) file.length()); + + try (FileInputStream fileInputStream = new FileInputStream(file)) { + compressedSize += InsertZipEntry(zipOutputStream, fileName, fileInputStream); + if (listener != null) { + listener.updateProgress(fileName, compressedSize); + } + } catch (IOException ioException) { + log.warn(String.format("Failed to add file to archive: %s", fileName), ioException); + } + } + zipOutputStream.flush(); + } + + private void validateNextFileWillNotReachMaxCompressedSize(double uncompressedSize) throws IOException { + if (maxSize > 0L && (double) compressedSize + uncompressedSize / AVERAGE_ZIP_COMPRESSION_RATIO > (double) maxSize) { + log.info("Maximum zip file size reached. Zip size: {} bytes Limit: {} bytes", compressedSize, maxSize); + throw new MaxZipSizeReached(compressedSize, maxSize); + } + } + + private long InsertZipEntry(ZipOutputStream zipOutputStream, String fileName, InputStream inputStream) throws IOException { + ZipEntry zipEntry = new ZipEntry(fileName); + zipOutputStream.putNextEntry(zipEntry); + IOUtils.copy(inputStream, zipOutputStream); + zipOutputStream.closeEntry(); + return zipEntry.getCompressedSize(); + } + + private DirectoryScanner createDirectoryScanner(File baseDir, String[] filterIncludePatterns, String[] filterExcludePatterns) { + DirectoryScanner ds = new DirectoryScanner(); + ds.setBasedir(baseDir); + ds.setCaseSensitive(false); + ds.setFollowSymlinks(true); + ds.setErrorOnMissingDir(false); + if (filterIncludePatterns != null && filterIncludePatterns.length > 0) { + ds.setIncludes(filterIncludePatterns); + } + + if (filterExcludePatterns != null && filterExcludePatterns.length > 0) { + ds.setExcludes(filterExcludePatterns); + } + + return ds; + } + + public long getFileCount() { + return fileCount; + } + + public static class MaxZipSizeReached extends IOException { + private long compressedSize; + private long maxZipSize; + + public MaxZipSizeReached(long compressedSize, long maxZipSize) { + super("Zip compressed size reached a limit of " + maxZipSize + " bytes"); + } + + public long getCompressedSize() { + return this.compressedSize; + } + + public long getMaxZipSize() { + return this.maxZipSize; + } + } + +} \ No newline at end of file diff --git a/src/main/java/com/cx/restclient/sast/utils/zip/Zipper.java b/src/main/java/com/cx/restclient/sast/utils/zip/Zipper.java index 02fcee48..f9c55c06 100644 --- a/src/main/java/com/cx/restclient/sast/utils/zip/Zipper.java +++ b/src/main/java/com/cx/restclient/sast/utils/zip/Zipper.java @@ -7,6 +7,7 @@ import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.ArrayUtils; import org.apache.tools.ant.DirectoryScanner; import org.apache.tools.zip.ZipEntry; import org.apache.tools.zip.ZipOutputStream; @@ -26,41 +27,39 @@ public Zipper(Logger log) { public void zip(File baseDir, String[] filterIncludePatterns, String[] filterExcludePatterns, OutputStream outputStream, long maxZipSize, ZipListener listener) throws IOException { assert baseDir != null : "baseDir must not be null"; - assert outputStream != null : "outputStream must not be null"; - DirectoryScanner ds = this.createDirectoryScanner(baseDir, filterIncludePatterns, filterExcludePatterns); - ds.setFollowSymlinks(true); + DirectoryScanner ds = createDirectoryScanner(baseDir, filterIncludePatterns, filterExcludePatterns); ds.scan(); - // this.printDebug(ds); + printDebug(ds); if (ds.getIncludedFiles().length == 0) { outputStream.close(); - log.info("No files to zip"); - throw new Zipper.NoFilesToZip(); - } else { - this.zipFile(baseDir, ds.getIncludedFiles(), outputStream, maxZipSize, listener); + log.debug("No files to zip"); + throw new NoFilesToZip(); } + + zipFile(baseDir, ds.getIncludedFiles(), outputStream, maxZipSize, listener); } private void zipFile(File baseDir, String[] files, OutputStream outputStream, long maxZipSize, ZipListener listener) throws IOException { - ZipOutputStream zipOutputStream = new ZipOutputStream(outputStream); - zipOutputStream.setEncoding("UTF8"); - long compressedSize = 0L; - double AVERAGE_ZIP_COMPRESSION_RATIO = 4.0D; - String[] arr$ = files; - int len$ = files.length; - - for (int i$ = 0; i$ < len$; ++i$) { - String fileName = arr$[i$]; - // log.debug("Adding file to zip: " + fileName); - File file = new File(baseDir, fileName); - if (!file.canRead()) { - log.warn("Skipping unreadable file: " + file); - } else { - if (maxZipSize > 0L && (double) compressedSize + (double) file.length() / 4.0D > (double) maxZipSize) { - log.info("Maximum zip file size reached. Zip size: " + compressedSize + " bytes Limit: " + maxZipSize + " bytes"); + try (ZipOutputStream zipOutputStream = new ZipOutputStream(outputStream)) { + zipOutputStream.setEncoding("UTF8"); + long compressedSize = 0; + final double AVERAGE_ZIP_COMPRESSION_RATIO = 4.0; + + for (String fileName : files) { + log.debug("Adding file to zip: " + fileName); + + File file = new File(baseDir, fileName); + if (!file.canRead()) { + log.debug("Skipping unreadable file: " + file); + continue; + } + + if (maxZipSize > 0 && compressedSize + (file.length() / AVERAGE_ZIP_COMPRESSION_RATIO) > maxZipSize) { + log.debug("Maximum zip file size reached. Zip size: " + compressedSize + " bytes Limit: " + maxZipSize + " bytes"); zipOutputStream.close(); - throw new Zipper.MaxZipSizeReached(compressedSize, maxZipSize); + throw new MaxZipSizeReached(compressedSize, maxZipSize); } if (listener != null) { @@ -69,22 +68,20 @@ private void zipFile(File baseDir, String[] files, OutputStream outputStream, lo ZipEntry zipEntry = new ZipEntry(fileName); zipOutputStream.putNextEntry(zipEntry); - FileInputStream fileInputStream = new FileInputStream(file); - IOUtils.copy(fileInputStream, zipOutputStream); - fileInputStream.close(); + try (FileInputStream fileInputStream = new FileInputStream(file)) { + IOUtils.copy(fileInputStream, zipOutputStream); + } zipOutputStream.closeEntry(); compressedSize += zipEntry.getCompressedSize(); } } - - zipOutputStream.close(); } private DirectoryScanner createDirectoryScanner(File baseDir, String[] filterIncludePatterns, String[] filterExcludePatterns) { DirectoryScanner ds = new DirectoryScanner(); ds.setBasedir(baseDir); ds.setCaseSensitive(false); - ds.setFollowSymlinks(false); + ds.setFollowSymlinks(true); ds.setErrorOnMissingDir(false); if (filterIncludePatterns != null && filterIncludePatterns.length > 0) { ds.setIncludes(filterIncludePatterns); diff --git a/src/main/java/com/cx/restclient/sca/dto/CreateProjectRequest.java b/src/main/java/com/cx/restclient/sca/dto/CreateProjectRequest.java new file mode 100644 index 00000000..30e8c6bc --- /dev/null +++ b/src/main/java/com/cx/restclient/sca/dto/CreateProjectRequest.java @@ -0,0 +1,13 @@ +package com.cx.restclient.sca.dto; + +public class CreateProjectRequest { + private String name; + + public void setName(String name) { + this.name = name; + } + + public String getName() { + return name; + } +} diff --git a/src/main/java/com/cx/restclient/sca/dto/CxSCAResolvingConfiguration.java b/src/main/java/com/cx/restclient/sca/dto/CxSCAResolvingConfiguration.java new file mode 100644 index 00000000..3606e581 --- /dev/null +++ b/src/main/java/com/cx/restclient/sca/dto/CxSCAResolvingConfiguration.java @@ -0,0 +1,23 @@ +package com.cx.restclient.sca.dto; + +import lombok.Getter; +import lombok.Setter; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +@Getter +@Setter +public class CxSCAResolvingConfiguration implements Serializable { + private List Manifests = new ArrayList<>(); + private List Fingerprints = new ArrayList<>(); + + public String getManifestsIncludePattern(){ + return String.join(",", Manifests); + } + + public String getFingerprintsIncludePattern(){ + return String.join(",", Fingerprints); + } +} diff --git a/src/main/java/com/cx/restclient/sca/dto/CxSCAScanAPIConfig.java b/src/main/java/com/cx/restclient/sca/dto/CxSCAScanAPIConfig.java new file mode 100644 index 00000000..95762901 --- /dev/null +++ b/src/main/java/com/cx/restclient/sca/dto/CxSCAScanAPIConfig.java @@ -0,0 +1,11 @@ +package com.cx.restclient.sca.dto; + +import lombok.Builder; +import lombok.Getter; + +@Builder +@Getter +public class CxSCAScanAPIConfig { + private boolean isZeroCodeZip; + private String includeSourceCode; +} diff --git a/src/main/java/com/cx/restclient/sca/dto/CxSCAScanApiConfigEntry.java b/src/main/java/com/cx/restclient/sca/dto/CxSCAScanApiConfigEntry.java new file mode 100644 index 00000000..abb5dad9 --- /dev/null +++ b/src/main/java/com/cx/restclient/sca/dto/CxSCAScanApiConfigEntry.java @@ -0,0 +1,10 @@ +package com.cx.restclient.sca.dto; +import lombok.Builder; +import lombok.Getter; + +@Builder +@Getter +public class CxSCAScanApiConfigEntry implements ScanAPIConfigEntry { + private String type; + private CxSCAScanAPIConfig value; +} diff --git a/src/main/java/com/cx/restclient/sca/dto/GetUploadUrlRequest.java b/src/main/java/com/cx/restclient/sca/dto/GetUploadUrlRequest.java new file mode 100644 index 00000000..8e1f75e4 --- /dev/null +++ b/src/main/java/com/cx/restclient/sca/dto/GetUploadUrlRequest.java @@ -0,0 +1,12 @@ +package com.cx.restclient.sca.dto; + +import lombok.Builder; +import lombok.Getter; + +import java.util.List; + +@Builder +@Getter +public class GetUploadUrlRequest { + private List config; +} diff --git a/src/main/java/com/cx/restclient/sca/dto/RemoteRepositoryInfo.java b/src/main/java/com/cx/restclient/sca/dto/RemoteRepositoryInfo.java new file mode 100644 index 00000000..a28211d2 --- /dev/null +++ b/src/main/java/com/cx/restclient/sca/dto/RemoteRepositoryInfo.java @@ -0,0 +1,20 @@ +package com.cx.restclient.sca.dto; + +import lombok.Getter; +import lombok.Setter; + +import java.io.Serializable; +import java.net.URL; + +/** + * Instructs SCA which repository should be scanned. + * In the future this class may be expanded to include repo credentials and commit/branch/tag reference. + */ +@Getter +@Setter +public class RemoteRepositoryInfo implements Serializable { + /** + * A URL for which 'git clone' is possible. + */ + private URL url; +} diff --git a/src/main/java/com/cx/restclient/sca/dto/ScanAPIConfigEntry.java b/src/main/java/com/cx/restclient/sca/dto/ScanAPIConfigEntry.java new file mode 100644 index 00000000..d19e80b0 --- /dev/null +++ b/src/main/java/com/cx/restclient/sca/dto/ScanAPIConfigEntry.java @@ -0,0 +1,4 @@ +package com.cx.restclient.sca.dto; + +public interface ScanAPIConfigEntry { +} diff --git a/src/main/java/com/cx/restclient/sca/utils/CxSCAFileSystemUtils.java b/src/main/java/com/cx/restclient/sca/utils/CxSCAFileSystemUtils.java new file mode 100644 index 00000000..8de35e57 --- /dev/null +++ b/src/main/java/com/cx/restclient/sca/utils/CxSCAFileSystemUtils.java @@ -0,0 +1,90 @@ +package com.cx.restclient.sca.utils; + +import com.cx.restclient.dto.PathFilter; +import org.apache.tools.ant.DirectoryScanner; +import org.slf4j.Logger; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.InvalidPathException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; + +public class CxSCAFileSystemUtils { + + public static String[] scanAndGetIncludedFiles(String baseDir, PathFilter filter) { + DirectoryScanner ds = createDirectoryScanner(new File(baseDir), filter.getIncludes(), filter.getExcludes()); + ds.setFollowSymlinks(true); + ds.scan(); + return ds.getIncludedFiles(); + } + + private static DirectoryScanner createDirectoryScanner(File baseDir, String[] filterIncludePatterns, String[] filterExcludePatterns) { + DirectoryScanner ds = new DirectoryScanner(); + ds.setBasedir(baseDir); + ds.setCaseSensitive(false); + ds.setFollowSymlinks(false); + ds.setErrorOnMissingDir(false); + + if (filterIncludePatterns != null && filterIncludePatterns.length > 0) { + ds.setIncludes(filterIncludePatterns); + } + + if (filterExcludePatterns != null && filterExcludePatterns.length > 0) { + ds.setExcludes(filterExcludePatterns); + } + + return ds; + } + + public static HashMap convertStringToKeyValueMap(String envString) { + + HashMap envMap = new HashMap<>(); + //"Key1:Val1,Key2:Val2" + String trimmedString = envString.replace("\"",""); + List envlist = Arrays.asList(trimmedString.split(",")); + + for( String pair : envlist) + { + String[] splitFromColon = pair.split(":",2); + String key = (splitFromColon[0]).trim(); + String value = (splitFromColon[1]).trim(); + envMap.put(key, value); + } + return envMap; + + } + + public static Path checkIfFileExists(String sourceDir, String fileString, String fileSystemSeparator, Logger log) + { + Path filePath = null; + try { + filePath = Paths.get(fileString); + if (Files.notExists(filePath) && !filePath.isAbsolute()) { + filePath = Paths.get(sourceDir, fileSystemSeparator, fileString); + if (Files.notExists(filePath)) { + log.info("File doesnt exist at the given location."); + filePath = null; + } + } + + } + catch (InvalidPathException e) + { + log.error("Invalid file path. Error Message :" + e.getMessage()); + } + catch (SecurityException e) + { + log.error("Unable to access the file. Error Message :" + e.getMessage()); + } + catch (Exception e) + { + log.error("Error while determing the existence of file. Error Message :" + e.getMessage()); + } + return filePath; + } + +} diff --git a/src/main/java/com/cx/restclient/sca/utils/fingerprints/CxSCAFileFingerprints.java b/src/main/java/com/cx/restclient/sca/utils/fingerprints/CxSCAFileFingerprints.java new file mode 100644 index 00000000..37ef82e0 --- /dev/null +++ b/src/main/java/com/cx/restclient/sca/utils/fingerprints/CxSCAFileFingerprints.java @@ -0,0 +1,51 @@ +package com.cx.restclient.sca.utils.fingerprints; + +import java.util.ArrayList; +import java.util.List; + +public class CxSCAFileFingerprints { + private String path; + private long size; + private List signatures = new ArrayList<>(); + + + public CxSCAFileFingerprints(String path, long size, List sig) { + this.path = path; + this.size = size; + this.signatures = sig; + } + + public CxSCAFileFingerprints(String path, long size) { + this.path = path; + this.size = size; + } + + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + + public long getSize() { + return size; + } + + public void setSize(long size) { + this.size = size; + } + + + public List getSignatures() { + return signatures; + } + + public void setSignatures(List signatures) { + this.signatures = signatures; + } + + public void addFileSignature(CxSCAFileSignature signature){ + this.signatures.add(signature); + } +} diff --git a/src/main/java/com/cx/restclient/sca/utils/fingerprints/CxSCAFileSignature.java b/src/main/java/com/cx/restclient/sca/utils/fingerprints/CxSCAFileSignature.java new file mode 100644 index 00000000..41ce1800 --- /dev/null +++ b/src/main/java/com/cx/restclient/sca/utils/fingerprints/CxSCAFileSignature.java @@ -0,0 +1,27 @@ +package com.cx.restclient.sca.utils.fingerprints; + +public class CxSCAFileSignature { + private String type; + private String value; + + public CxSCAFileSignature(String type, String value) { + this.type = type; + this.value = value; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } +} diff --git a/src/main/java/com/cx/restclient/sca/utils/fingerprints/CxSCAScanFingerprints.java b/src/main/java/com/cx/restclient/sca/utils/fingerprints/CxSCAScanFingerprints.java new file mode 100644 index 00000000..27eafee8 --- /dev/null +++ b/src/main/java/com/cx/restclient/sca/utils/fingerprints/CxSCAScanFingerprints.java @@ -0,0 +1,40 @@ +package com.cx.restclient.sca.utils.fingerprints; + +import java.util.ArrayList; +import java.util.List; + +public class CxSCAScanFingerprints { + + private String version; + private String time; + private List fingerprints = new ArrayList<>(); + + + public CxSCAScanFingerprints(){ + version = "1.0.0"; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public List getFingerprints() { + return fingerprints; + } + + public void addFileFingerprints(CxSCAFileFingerprints fileFingerprints){ + fingerprints.add(fileFingerprints); + } + + public String getTime() { + return time; + } + + public void setTime(String time) { + this.time = time; + } +} diff --git a/src/main/java/com/cx/restclient/sca/utils/fingerprints/FingerprintCollector.java b/src/main/java/com/cx/restclient/sca/utils/fingerprints/FingerprintCollector.java new file mode 100644 index 00000000..858d8ac4 --- /dev/null +++ b/src/main/java/com/cx/restclient/sca/utils/fingerprints/FingerprintCollector.java @@ -0,0 +1,81 @@ +package com.cx.restclient.sca.utils.fingerprints; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.slf4j.Logger; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.List; + + +public class FingerprintCollector { + + private static final String DEFAULT_FINGERPRINT_FILENAME = "CxSCAFingerprints.json"; + private final SignatureCalculator sha1SignatureCalculator; + private final Logger log; + + public FingerprintCollector(Logger log){ + this.log = log; + sha1SignatureCalculator = new Sha1SignatureCalculator(); + } + + public CxSCAScanFingerprints collectFingerprints(String baseDir, + List files) { + log.info(String.format("Started fingerprint collection on %s", baseDir)); + + CxSCAScanFingerprints scanFingerprints = new CxSCAScanFingerprints(); + + + for (String filePath : files) { + + Path fullFilePath = Paths.get(baseDir, filePath); + try { + log.debug(String.format("Calculating signatures for file %s", fullFilePath)); + byte[] fileContent = Files.readAllBytes(fullFilePath); + CxSCAFileFingerprints fingerprints = new CxSCAFileFingerprints(filePath, Files.size(fullFilePath)); + + fingerprints.addFileSignature(sha1SignatureCalculator.calculateSignature(fileContent)); + + scanFingerprints.addFileFingerprints(fingerprints); + } catch (IOException e) { + log.error(String.format("Failed calculating file signature: %s",fullFilePath.toString() ), e); + } + } + log.info(String.format("Calculated fingerprints for %d files", scanFingerprints.getFingerprints().size())); + scanFingerprints.setTime(LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"))); + return scanFingerprints; + } + + public void writeScanFingerprintsFile(CxSCAScanFingerprints scanFingerprints, String path) throws IOException { + + long fingerprintCount = scanFingerprints.getFingerprints().size(); + if (fingerprintCount == 0){ + log.info("No supported files for fingerprinting found in this scan"); + } + String fingerprintFilePath = path; + File targetLocation = new File(path); + if (targetLocation.isDirectory()){ + fingerprintFilePath = Paths.get(path, DEFAULT_FINGERPRINT_FILENAME).toString(); + } + + log.info(String.format("Writing %d file signatures to fingerprint file: %s", fingerprintCount, fingerprintFilePath)); + ObjectMapper objectMapper = new ObjectMapper(); + File fingerprintFile = new File(fingerprintFilePath); + objectMapper.writeValue(fingerprintFile, scanFingerprints); + + } + + + public static String getFingerprintsAsJsonString(CxSCAScanFingerprints scanFingerprints) throws JsonProcessingException { + ObjectMapper objectMapper = new ObjectMapper(); + return objectMapper.writeValueAsString(scanFingerprints); + } + + +} diff --git a/src/main/java/com/cx/restclient/sca/utils/fingerprints/Sha1SignatureCalculator.java b/src/main/java/com/cx/restclient/sca/utils/fingerprints/Sha1SignatureCalculator.java new file mode 100644 index 00000000..ae743515 --- /dev/null +++ b/src/main/java/com/cx/restclient/sca/utils/fingerprints/Sha1SignatureCalculator.java @@ -0,0 +1,28 @@ +package com.cx.restclient.sca.utils.fingerprints; + +import com.cx.restclient.exception.CxClientException; +import org.apache.commons.codec.binary.Hex; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +public class Sha1SignatureCalculator implements SignatureCalculator { + + + private static final String SHA1_SIGNATURE_TYPE_NAME = "SHA1"; + + @Override + public CxSCAFileSignature calculateSignature(byte[] content) throws CxClientException { + MessageDigest digest; + try { + digest = MessageDigest.getInstance("SHA-1"); + } catch (NoSuchAlgorithmException e) { + throw new CxClientException("Unable to use SHA-1 algorithm", e); + } + + digest.update(content); + + return new CxSCAFileSignature(SHA1_SIGNATURE_TYPE_NAME, Hex.encodeHexString(digest.digest())); + } + +} diff --git a/src/main/java/com/cx/restclient/sca/utils/fingerprints/SignatureCalculator.java b/src/main/java/com/cx/restclient/sca/utils/fingerprints/SignatureCalculator.java new file mode 100644 index 00000000..66b8766e --- /dev/null +++ b/src/main/java/com/cx/restclient/sca/utils/fingerprints/SignatureCalculator.java @@ -0,0 +1,11 @@ +package com.cx.restclient.sca.utils.fingerprints; + +import com.cx.restclient.exception.CxClientException; + +import java.io.IOException; + + +public interface SignatureCalculator { + CxSCAFileSignature calculateSignature(byte[] content) throws IOException, CxClientException; + +} diff --git a/src/main/resources/com/cx/report/report.ftl b/src/main/resources/com/cx/report/report.ftl index afc4f8cb..f58787fa 100644 --- a/src/main/resources/com/cx/report/report.ftl +++ b/src/main/resources/com/cx/report/report.ftl @@ -34,7 +34,7 @@ background-color: #372F51; } - .cx-report .legend-color-box.new-legend-color { + .cx-report .legend-color-box.new-legend-color { background: linear-gradient(45deg, white 25%, #373050 25%, #373050 50%, white 50%, white 75%, #373050 75%); background-size: 4px 4px; } @@ -371,10 +371,9 @@ background-color: #373050; } - - .threshold-exceeded, - .threshold-compliance, - .policy-compliance{ + .threshold-exceeded, + .threshold-compliance, + .policy-compliance { min-width: 100%; display: inline-flex; font-size: 14px; @@ -383,21 +382,21 @@ padding: 4px 9px; } - .threshold-exceeded { + .threshold-exceeded { background-color: #DA2945; color: white; border-radius: 2px; font-weight: bold; } - .policy-compliance{ + + .policy-compliance { border-radius: 2px; font-weight: bold; } - .threshold-exceeded-icon, .threshold-compliance-icon, - .policy-compliance{ + .policy-compliance { display: inline-flex; padding-right: 6px; margin: auto 0; @@ -426,11 +425,10 @@ overflow: hidden; } - .cx-report .results-report .libraries-vulnerable-number{ + .cx-report .results-report .libraries-vulnerable-number { font-size: 18px; } - .cx-report .results-report .libraries-vulnerable-text { font-size: 12px; line-height: 15px; @@ -627,9 +625,10 @@ .cx-report .html-report.download-icon { margin-right: 6px; } - .cx-report .pdf-report.download-icon, { + + .cx-report .pdf-report.download-icon { margin-right: 6px; - border-left: solid 1px #d5d5d + border-left: 1px solid #d5d5d5; } .cx-report .summary-section .html-report { @@ -808,6 +807,7 @@ margin-top: -3px; margin-left: -7px; } + .scan-status .content-scan-status li { margin-top: 6px; margin-bottom: 6px; @@ -826,7 +826,6 @@ padding-top: 10px; } - .scan-status .indicator-scan-status.success { background-color: #38d87d; } @@ -839,7 +838,7 @@ margin-top: 5px; text-align: center; border: 1px solid #ffffff; - padding-left: 5px ; + padding-left: 5px; } .scan-status .indicator-scan-status.success .icon-scan-status { @@ -850,23 +849,20 @@ border: 1px solid #DD3D56; } - - - .scan-status .title-scan-status { font-size: 12px; font-weight: bold; padding-left: 16px; + padding-top: 10px; } - .scan-status .title-scan-status.failure { + .scan-status .content-scan-status .title-scan-status.failure { padding-top: 7px; color: #DD3D56; padding-bottom: 3px; } - } - .scan-status .title-scan-status.success { + .scan-status .content-scan-status .title-scan-status.success { color: #38d87d; } @@ -909,91 +905,106 @@ -
Checkmarx Report
- <#if buildFailed> -
-
-
- - - error - Created with Sketch. - - - - - - - - - - - - - - - -
-
-
-

- Checkmarx Scan Failed -

-
    - <#if policyViolated> -
  • ${osa.osaPolicies?size} ${policyLabel} Violated
    • - - <#if config.sastEnabled && sast.sastResultsReady && (sastThresholdExceeded || sastNewResultsExceeded) && config.osaEnabled && osa.osaResultsReady && osaThresholdExceeded> -
  • CxSAST and CxOSA Vulnerability Thresholds Exceeded
    • - <#elseif config.sastEnabled && sast.sastResultsReady && (sastThresholdExceeded || sastNewResultsExceeded)> -
  • CxSAST Vulnerability Threshold Exceeded
    • - <#elseif config.osaEnabled && osa.osaResultsReady && osaThresholdExceeded> -
  • CxOSA Vulnerability Threshold Exceeded
    • - -
-
-
- <#else> -
-
-
- - - OK - Created with Sketch. - - - - - - - - - - - - - - - - - - -
-
-
-

- Checkmarx Scan Passed -

-
-
- - + <#if buildFailed> +
+
+
+ + + error + Created with Sketch. + + + + + + + + + + + + + + + +
+
+
+

+ Checkmarx scan found the following issues: +

+
    + <#if config.isSastEnabled() && !sast.sastResultsReady> +
  • SAST Scan Failed
    • + + <#if config.isOsaEnabled() && !dependencyResult.resultReady> +
  • OSA Scan Failed
    • + + <#if policyViolated> +
  • ${policyViolatedCount} ${policyLabel} Violated
    • + + <#if config.isSastEnabled() && sast.sastResultsReady && (sastThresholdExceeded || sastNewResultsExceeded) && (config.isOsaEnabled() || config.isAstScaEnabled()) && dependencyResult.resultReady && dependencyThresholdExceeded> +
  • Exceeded CxSAST and CxOSA/CxSCA Vulnerability Thresholds
    • + <#elseif config.isSastEnabled() && sast.sastResultsReady && (sastThresholdExceeded || sastNewResultsExceeded)> +
  • Exceeded CxSAST Vulnerability Threshold
    • + <#elseif config.isOsaEnabled() && dependencyResult.resultReady && dependencyThresholdExceeded> +
  • Exceeded CxOSA Vulnerability Threshold
    • + <#elseif config.isAstScaEnabled() && dependencyResult.resultReady && dependencyThresholdExceeded> +
  • Exceeded CxSCA Vulnerability Threshold
    • + <#else> +
  • CxScan Failed
    • + +
+
+
+ <#else> +
+
+
+ + + OK + Created with Sketch. + + + + + + + + + + + + + + + + + + +
+
+
+

+ Checkmarx Scan Passed +

+
+
+ +
@@ -1002,8 +1013,8 @@
- <#if config.sastEnabled> -
+ <#if config.isSastEnabled()> +
CxSAST Vulnerabilities Status
<#if sast.sastResultsReady> @@ -1013,7 +1024,7 @@
- viewBox="0 0 12 14"> + viewBox="0 0 12 14"> @@ -1299,288 +1310,555 @@
- <#if config.osaEnabled> -
-
-
CxOSA Vulnerabilities & Libraries
- <#if osa.osaResultsReady> -