.travis.yml

# Copyright (c) 2013-2019 Arm Limited. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the License); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an AS IS BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

language: sh
os: linux
dist: xenial


env:
  global:
    - deps_url="https://mbed-os-ci.s3-eu-west-1.amazonaws.com/jenkins-ci/deps"
    - deps_dir="${HOME}/.cache/deps"

cache:
  pip: true
  directories:
    - ${HOME}/.cache/deps
    - ${HOME}/.ccache


before_install:
  - source tools/test/travis-ci/functions.sh
  - set_status "pending" "Test started."

after_success:
  - set_status "success" "Success!"

after_failure:
  - set_status "failure" "Test failed."


matrix:
  include:

    ### Basic Tests ###
    - &basic-vm
      stage: "Basic"
      name: "file attributes"
      env: NAME=gitattributestest
      script:
        - git diff --exit-code

    - <<: *basic-vm
      name: "license check"
      env: NAME=licence_check
      language: python
      python: 3.6.8 # scancode-toolkit v3.1.1 requires v3.6.8
      install:
        # workaround for https://github.com/ARMmbed/mbed-os/issues/13322
        - pip install pdfminer.six==20200517
        - pip install scancode-toolkit==3.1.1
      before_script:
        - mkdir -p SCANCODE
        - mkdir -p SCANCODE_NEW_FILES
        # Fetch remaining information needed for branch comparison
        - git fetch --all --unshallow --tags
        - git fetch origin "${TRAVIS_BRANCH}"
      script:
        # scancode does not support list of files, only one file or directory
        # we use SCANCODE directory for all changed files (their copies with full tree)
        - >-
            git diff --name-only --diff-filter=ad FETCH_HEAD..HEAD \
              | ( grep '.\(c\|cpp\|h\|hpp\|py\)$' || true ) \
              | ( grep -v '^tools/test/toolchains/api_test.py' || true ) \
              | while read file; do cp --parents "${file}" SCANCODE; done
        - scancode -l --json-pp scancode.json SCANCODE
        - python ./tools/test/travis-ci/scancode-evaluate.py -f scancode.json || true
        # run the same but for new files. All new files must have SPDX
        - >-
            git diff --name-only --diff-filter=A FETCH_HEAD..HEAD \
              | ( grep '.\(c\|cpp\|h\|hpp\|py\)$' || true ) \
              | ( grep -v '^tools/test/toolchains/api_test.py' || true ) \
              | while read file; do cp --parents "${file}" SCANCODE_NEW_FILES; done
        - scancode -l --json-pp scancode_new_files.json SCANCODE_NEW_FILES
        - python ./tools/test/travis-ci/scancode-evaluate.py -f scancode_new_files.json || true
      after_success:
        - python ./tools/test/travis-ci/scancode-evaluate.py -f scancode.json
        - cat scancode-evaluate.log
        - COUNT=$(cat scancode-evaluate.log | grep 'File:' | wc -l)
        - python ./tools/test/travis-ci/scancode-evaluate.py -f scancode_new_files.json
        - cat scancode-evaluate.log
        - COUNT_NEW_FILES=$(cat scancode-evaluate.log | grep 'File:' | wc -l)
        - |
          if [ $COUNT == 0 ] && [ $COUNT_NEW_FILES == 0 ]; then
              echo "License check OK";
              STATUSM="All licenses OK";
              set_status "success" "$STATUSM";
            elif [ $COUNT_NEW_FILES != 0 ]; then
              echo "License check failed, files with the license issues found";
              STATUSM="Needs review, license issues in modified files: ${COUNT}, new files: ${COUNT_NEW_FILES}";
              set_status "failure" "$STATUSM";
              false;
            else
              echo "License check failed, please review license issues found in modified files";
              STATUSM="Needs review, ${COUNT} license issues found";
              set_status "success" "$STATUSM";
              false;
          fi



    - <<: *basic-vm
      name: "include check"
      env: NAME=include_check
      script:
        - |
          ! git grep '^#include\s["'"']mbed.h['"'"]$' -- '*.c' '*.h' '*.cpp' '*.hpp' \
              ':!*platform_mbed.h' ':!*TESTS/*' ':!TEST_APPS/' ':!UNITTESTS/' \
              ':!*tests/*' ':!*targets/*' ':!*TARGET_*' ':!*unsupported/*' \
              ':!*events/tests/*' ':!*drivers/tests/*'


    ### Docs Tests ###
    - &docs-vm
      stage: "Docs"
      name: "astyle"
      env: NAME=astyle
      install:
        - >-
          curl -L0 https://mbed-os.s3-eu-west-1.amazonaws.com/builds/deps/astyle_3.1_linux.tar.gz --output astyle.tar.gz;
          mkdir -p BUILD && tar xf astyle.tar.gz -C BUILD;
          cd BUILD/astyle/build/gcc;
          make;
          export PATH="${PWD}/bin:${PATH}";
          cd -
        - astyle --version
        # Fetch remaining information needed for branch comparison
        - git fetch --all --unshallow --tags
        - git fetch origin "${TRAVIS_BRANCH}"
      script:
        - >-
          git diff --name-only --diff-filter=d FETCH_HEAD..HEAD \
            | ( grep '.*\.\(c\|cpp\|h\|hpp\)$' || true ) \
            | ( grep -v -f .codecheckignore || true ) \
            | while read file; do astyle -n --options=.astylerc "${file}"; done
        - git diff --exit-code --diff-filter=d --color

    - <<: *docs-vm
      name: "spellcheck"
      env: NAME=doxy-spellcheck
      install:
        - source_pkg aspell
      script:
        # TODO: run checks on all directories once all mispellings are fixed
        - ./tools/test/travis-ci/doxy-spellchecker/spell.sh drivers .codecheckignore
        - ./tools/test/travis-ci/doxy-spellchecker/spell.sh platform .codecheckignore
        - ./tools/test/travis-ci/doxy-spellchecker/spell.sh events .codecheckignore
        - ./tools/test/travis-ci/doxy-spellchecker/spell.sh rtos .codecheckignore
        - ./tools/test/travis-ci/doxy-spellchecker/spell.sh connectivity/netsocket .codecheckignore

    - <<: *docs-vm
      name: "doxygen"
      env: NAME=docs
      install:
        # Build doxygen
        - >
          (git clone --depth=1 --single-branch --branch Release_1_8_14 https://github.com/doxygen/doxygen;
          cd doxygen;
          mkdir build;
          cd build;
          cmake -G "Unix Makefiles" ..;
          make;
          sudo make install)
        # Create BUILD directory for tests
        - ccache -s
        - mkdir BUILD
      script:
        # Assert that the Doxygen build produced no warnings.
        # The strange command below asserts that the Doxygen command had an
        # output of zero length
        - doxygen doxyfile_options 2>&1
        # Once Mbed OS has been fixed, enable the full test by replacing the top line with this:
        # - ( ! doxygen doxyfile_options 2>&1 | grep . )
        # Assert that all binary libraries are named correctly
        # The strange command below asserts that there are exactly 0 libraries
        # that do not start with lib
        - >
          find "(" -name "*.a" -or -name "*.ar" ")" -and -not -name "lib*" |
          tee BUILD/badlibs |
          sed -e "s/^/Bad library name found: /" && [ ! -s BUILD/badlibs ]
        # Assert that all assembler files are named correctly
        # The strange command below asserts that there are exactly 0 libraries
        # that do end with .s
        - >
          find  -name "*.s" | tee BUILD/badasm |
          sed -e "s/^/Bad Assembler file name found: /" && [ ! -s BUILD/badasm ]

    - <<: *docs-vm
      name: "Frozen tools check"
      env: NAME=frozen_tools_check
      script:
        # Reject any changes to tools that would require a re-release of the
        # tools for the online compiler.
        - >-
            frozen_files=`\
                git diff --name-only --diff-filter=d FETCH_HEAD..HEAD \
                    | egrep \
                      -e "^tools/build_api*" \
                      -e "^tools/config*" \
                      -e "^tools/export*" \
                      -e "^tools/notifier*" \
                      -e "^tools/paths*" \
                      -e "^tools/resources*" \
                      -e "^tools/targets*" \
                      -e "^tools/toolchains*" \
                      -e "^tools/utils*" \
                      -e "^$"`
            if [ -z "$frozen_files" ]; then
                echo "Success!";
            else
                echo -e "Failure: Frozen files were modified\n$frozen_files";
                echo -e "Please see https://os.mbed.com/blog/entry/Introducing-the-new-Mbed-Tools/" \
                        "\nfor why we've frozen the legacy tools.";
                false;
            fi

    ### Python Tests ###
    - &pytools-vm
      stage: "Pytest"
      name: "tools-py27"
      env: NAME=tools-py2.7
      language: python
      python: 2.7
      install:
        # Install gcc
        - source_pkg gcc
        - arm-none-eabi-gcc --version
        # Install additional python modules
        - python --version
        - |-
          tr -d ' ' >> requirements.txt <<< "
          mock==2.0.0
          pytest==3.3.0
          pylint>=1.9,<2
          hypothesis>=3,<4
          coverage>=4.5,<5
          "
        - python -m pip install --upgrade pip==18.1
        - python -m pip install --upgrade setuptools==40.4.3
        - pip install -r requirements.txt
        - pip list --verbose
      script:
        # Run local testing on tools
        - PYTHONPATH=. coverage run -a -m pytest tools/test
        - python tools/test/pylint.py
        - coverage run -a tools/project.py -S | sed -n '/^Total/p'
        - coverage html

    - <<: *pytools-vm
      name: "tools-py35"
      env: NAME=tools-py3.5
      python: 3.5

    - <<: *pytools-vm
      name: "tools-py36"
      env: NAME=tools-py3.6
      python: 3.6

    - <<: *pytools-vm
      name: "tools-py37"
      env: NAME=tools-py3.7
      python: 3.7


    ### Extended Tests ###
    - &extended-vm
      stage: "Extended"
      name: "events"
      env: NAME=events EVENTS=events
      language: python
      python: 3.7
      install:
        # Install gcc
        - source_pkg gcc
        - arm-none-eabi-gcc --version
        # Install python modules
        - python -m pip install --upgrade pip==18.1
        - python -m pip install --upgrade setuptools==40.4.3
        - pip install -r requirements.txt
        - pip list --verbose
      script:
        # Check that example compiles
        - sed -n '/``` cpp/,/```/{/```$/Q;/```/d;p;}' ${EVENTS}/README.md > main.cpp
        - python tools/make.py -t GCC_ARM -m K64F --source=. --build=BUILD/K64F/GCC_ARM -j0
        # Check that example compiles without rtos
        - sed -n '/``` cpp/,/```/{/```$/Q;/```/d;p;}' ${EVENTS}/README.md > main.cpp
        - |
          rm -r cmsis/CMSIS_5/CMSIS/RTOS2 cmsis/device/rtos drivers/source/usb connectivity/cellular connectivity/drivers/cellular \
            connectivity/libraries/ppp connectivity/netsocket connectivity/nanostack connectivity/lwipstack features/frameworks/greentea-client \
            features/frameworks/utest features/frameworks/unity BUILD
        - python tools/make.py -t GCC_ARM -m NUCLEO_F103RB --source=. --build=BUILD/NUCLEO_F103RB/GCC_ARM -j0
        # Run profiling tests
        - make -C ${EVENTS}/tests/unit prof | tee prof
        - ccache -s
      after_success:
        # Update status, comparing with master if possible.
        - |
          CURR=$(grep -o '[0-9]\+ cycles' prof | awk '{sum += $1} END {print sum}')
          PREV=$(curl -u "${MBED_BOT}" https://api.github.com/repos/${TRAVIS_REPO_SLUG}/status/master \
              | jq -re "select(.sha != \"${TRAVIS_COMMIT}\")
                  | .statuses[] | select(.context == \"travis-ci/${NAME}\").description
                  | capture(\"runtime is (?<runtime>[0-9]+)\").runtime" \
              || echo 0)

          delta=""
          [ "${PREV}" -ne 0 ] && delta="($(printf "%+d" "$(( ${CURR} - ${PREV} ))" cycles)"

          set_status "success" "Success! Runtime is ${CURR} cycles. ${delta}"

    - <<: *extended-vm
      name: "littlefs"
      env: NAME=littlefs LITTLEFS=storage/filesystem/littlefs
      install:
        # Install gcc
        - source_pkg gcc
        - arm-none-eabi-gcc --version
        # Install python modules
        - python -m pip install --upgrade pip==18.1
        - python -m pip install --upgrade setuptools==40.4.3
        - pip install -r requirements.txt
        - pip list --verbose
        # Install test-specific packages
        - source_pkg fuse
        - source_pkg libfuse-dev
        - fusermount --version
      before_script:
        # Setup and patch littlefs-fuse
        - git clone https://github.com/armmbed/littlefs-fuse littlefs_fuse
        - git -C littlefs_fuse checkout 3f1ed6e37799e49e3710830dc6abb926d5503cf2
        - echo '*' > littlefs_fuse/.mbedignore
        - rm -rf littlefs_fuse/littlefs/*
        - cp -r $(git ls-tree --name-only HEAD ${LITTLEFS}/littlefs/) littlefs_fuse/littlefs
        # Create file-backed disk
        - mkdir MOUNT
        - sudo chmod a+rw /dev/loop0
        - dd if=/dev/zero bs=512 count=2048 of=DISK
        - losetup /dev/loop0 DISK
        - CFLAGS="-Werror -Wno-format"
      script:
        # Check that example compiles
        - export CFLAGS="-Werror -Wno-format"
        - sed -n '/``` c++/,/```/{/```/d;p;}' ${LITTLEFS}/README.md > main.cpp
        - python tools/make.py -t GCC_ARM -m K82F --source=. --build=BUILD/K82F/GCC_ARM -j0
        # Run local littlefs tests
        - make -C${LITTLEFS}/littlefs test QUIET=1
        # Run local littlefs tests with set of variations
        - make -C${LITTLEFS}/littlefs test QUIET=1 CFLAGS+="-DLFS_READ_SIZE=64  -DLFS_PROG_SIZE=64"
        - make -C${LITTLEFS}/littlefs test QUIET=1 CFLAGS+="-DLFS_READ_SIZE=1   -DLFS_PROG_SIZE=1"
        - make -C${LITTLEFS}/littlefs test QUIET=1 CFLAGS+="-DLFS_READ_SIZE=512 -DLFS_PROG_SIZE=512"
        - make -C${LITTLEFS}/littlefs test QUIET=1 CFLAGS+="-DLFS_BLOCK_COUNT=1023 -DLFS_LOOKAHEAD=2048"
        - make -C${LITTLEFS}/littlefs clean test QUIET=1 CFLAGS+="-DLFS_NO_INTRINSICS"
        # Self-hosting littlefs fuzz test with littlefs-fuse
        - make -Clittlefs_fuse
        - littlefs_fuse/lfs --format /dev/loop0
        - littlefs_fuse/lfs /dev/loop0 MOUNT
        - ls MOUNT
        - mkdir MOUNT/littlefs
        - cp -r $(git ls-tree --name-only HEAD ${LITTLEFS}/littlefs/) MOUNT/littlefs
        - ls MOUNT/littlefs
        - CFLAGS="-Wno-format" make -CMOUNT/littlefs -B test_dirs test_files QUIET=1
        # Compile and find the code size with smallest configuration
        - cd ${TRAVIS_BUILD_DIR}/${LITTLEFS}/littlefs
        - make clean size
            CC='arm-none-eabi-gcc -mthumb'
            OBJ="$(ls lfs*.o | tr '\n' ' ')"
            CFLAGS+="-DLFS_NO_ASSERT -DLFS_NO_DEBUG -DLFS_NO_WARN -DLFS_NO_ERROR"
            | tee sizes
        - ccache -s
      after_success:
        # Update status, comparing with master if possible.
        - |
          CURR=$(tail -n1 sizes | awk '{print $1}')
          PREV=$(curl -u "${MBED_BOT}" https://api.github.com/repos/${TRAVIS_REPO_SLUG}/status/master \
              | jq -re "select(.sha != \"${TRAVIS_COMMIT}\")
                  | .statuses[] | select(.context == \"travis-ci/${NAME}\").description
                  | capture(\"code size is (?<size>[0-9]+)\").size" \
              || echo 0)

          delta=""
          [ "${PREV}" -ne 0 ] && delta="($(printf "%+0.2f%%" "$(<<< "100 * ((${CURR} - ${PREV})/${PREV})" bc -l)"))"

          set_status "success" "Success! Code size is ${CURR}B. ${delta}"