Add a script to run Travis CI locally.

This isn't quite Travis, but close enough for local testing.

.restyled.yaml

@@ -2,4 +2,10 @@
 restylers:
   - astyle:
       arguments: ["--options=other/astyle/astylerc"]
+  - autopep8
+  - black
   - prettier-yaml
+  - reorder-python-imports
+  - shellharden
+  - shfmt
+  - yapf

.travis.yml

@@ -23,7 +23,7 @@ jobs:
             - libgtest-dev           # For unit tests.
             - libvpx-dev             # For toxav.
             - ninja-build
-            - pylint
+            - pylint3
       install:       .travis/$JOB install
       script:        .travis/$JOB script
       after_script:  .travis/upload-coverage

.travis/cmake-linux

@@ -5,7 +5,7 @@ ACTION="$1"
 set -eu
 
 CACHEDIR="$HOME/cache"
-NPROC=`nproc`
+NPROC=$(nproc)
 ASTYLE="$CACHEDIR/astyle/build/gcc/bin/astyle"
 ASTYLE_VERSION=3.1
 TRAVIS_TOOL="https://raw.githubusercontent.com/TokTok/ci-tools/master/bin/travis-haskell"
@@ -21,7 +21,7 @@ travis_install() {
     # Work around https://github.com/eddyxu/cpp-coveralls/issues/108 by manually
     # installing the pyOpenSSL module and injecting it into urllib3 as per
     # https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl-py2
-    sed -i -e '/^import sys$/a import urllib3.contrib.pyopenssl\nurllib3.contrib.pyopenssl.inject_into_urllib3()' `which coveralls`
+    sed -i -e '/^import sys$/a import urllib3.contrib.pyopenssl\nurllib3.contrib.pyopenssl.inject_into_urllib3()' "$(which coveralls)"
   }
 
   # Install astyle (version in ubuntu-precise too old).
@@ -34,15 +34,15 @@ travis_install() {
 }
 
 run_static_analysis() {
-  pylint -E other/analysis/check_recursion
+  pylint3 -E other/analysis/check_recursion
 
   export CPPFLAGS="-isystem $CACHEDIR/include"
   export LDFLAGS="-L$CACHEDIR/lib"
-  cat toxav/*.c toxcore/*.c toxencryptsave/*.c \
-    | clang `pkg-config --cflags libsodium opus vpx` \
-        -Itoxav -Itoxcore -Itoxencryptsave -S -emit-llvm -xc - -o- \
-    | opt -analyze -print-callgraph 2>&1 \
-    | other/analysis/check_recursion
+  cat toxav/*.c toxcore/*.c toxencryptsave/*.c |
+    clang "$(pkg-config --cflags libsodium opus vpx)" \
+      -Itoxav -Itoxcore -Itoxencryptsave -S -emit-llvm -xc - -o- |
+    opt -analyze -print-callgraph 2>&1 |
+    other/analysis/check_recursion
   other/analysis/run-clang
   other/analysis/run-clang-analyze
 }
@@ -85,10 +85,10 @@ travis_script() {
 
   cmake --build _build --parallel "$NPROC" --target install -- -k 0
 
-  cd _build  # pushd
-  ctest -j50 --output-on-failure || \
+  cd _build # pushd
+  ctest -j50 --output-on-failure ||
     ctest -j50 --output-on-failure --rerun-failed
-  cd -  # popd
+  cd - # popd
 }
 
 if [ "-z" "$ACTION" ]; then

other/analysis/check_recursion

@@ -1,25 +1,26 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 """
 Tool to check for recursive calls in toxcore C code.
 
-Usage: 
+Usage:
 
 cat toxav/*.c toxcore/*.c toxencryptsave/*.c \
   | clang `pkg-config --cflags libsodium opus vpx` \
       -Itoxav -Itoxcore -Itoxencryptsave -S -emit-llvm -xc - -o- \
   | opt -analyze -print-callgraph 2>&1 \
   | other/analysis/check_recursion
 """
-
-from __future__ import print_function
-
 import collections
 import fileinput
 import re
 import sys
 import time
+from typing import Dict
+from typing import List
+from typing import Set
 
-def load_callgraph():
+
+def load_callgraph() -> Dict:
     """
     Parses the output from opt -print-callgraph from stdin or argv.
 
@@ -39,7 +40,14 @@ def load_callgraph():
 
     return {k: sorted(v) for k, v in graph.items()}
 
-def walk(visited, callgraph, cycles, stack, cur):
+
+def walk(
+        visited: Set,
+        callgraph: Dict,
+        cycles: Set,
+        stack: List,
+        cur: str,
+) -> None:
     """
     Detects cycles in the callgraph and adds them to the cycles parameter.
     """
@@ -54,13 +62,15 @@ def walk(visited, callgraph, cycles, stack, cur):
             visited.add(callee)
     stack.pop()
 
-def get_time():
+
+def get_time() -> int:
     """
     Return the current time in milliseconds.
     """
     return int(round(time.time() * 1000))
 
-def find_recursion(expected):
+
+def find_recursion(expected: Set) -> None:
     """
     Main function: detects cycles and prints them.
 
@@ -98,6 +108,7 @@ def find_recursion(expected):
     if expected or cycles:
         sys.exit(1)
 
+
 find_recursion(expected={
     "add_to_closest -> add_to_closest",
     "add_to_list -> add_to_list",

other/astyle/format-source

@@ -36,10 +36,11 @@ FROM_JSON='s/\\"/"/g;s/^"(.*)"$/$1/;s/\\\\/\\/g;s/\\n/\n/g'
 apidsl_request() {
   TMPFILE=$(mktemp /tmp/apidsl.XXXXXX)
   curl -s -o "$TMPFILE" -X POST --data @<(
-    echo '["Request",';
-    cat $2;
-    echo ']') https://apidsl.herokuapp.com/$1
-  if grep '\[1,"' "$TMPFILE" > /dev/null; then
+    echo '["Request",'
+    cat "$2"
+    echo ']'
+  ) "https://apidsl.herokuapp.com/$1"
+  if grep '\[1,"' "$TMPFILE" >/dev/null; then
     echo "Error: $(grep -o '".*"' /tmp/apidsl-$$ | perl -0777 -pe "$FROM_JSON")" >&2
     rm "$TMPFILE"
     exit 1
@@ -52,34 +53,42 @@ apidsl_curl() {
   echo "apidsl_curl $*" >&2
   apidsl_request "c" <(
     apidsl_request "parse" <(
-      perl -0777 -pe "$TO_JSON" $1)) | perl -0777 -pe "$FROM_JSON"
+      perl -0777 -pe "$TO_JSON" "$1"
+    )
+  ) | perl -0777 -pe "$FROM_JSON"
 }
 
 # Check if apidsl generated sources are up to date.
 set +x
-$APIDSL toxcore/LAN_discovery.api.h > toxcore/LAN_discovery.h &
-$APIDSL toxcore/crypto_core.api.h > toxcore/crypto_core.h &
-$APIDSL toxcore/ping.api.h > toxcore/ping.h &
-$APIDSL toxcore/ping_array.api.h > toxcore/ping_array.h &
-$APIDSL toxcore/tox.api.h > toxcore/tox.h &
-$APIDSL toxav/toxav.api.h > toxav/toxav.h &
-$APIDSL toxencryptsave/toxencryptsave.api.h > toxencryptsave/toxencryptsave.h &
+"$APIDSL" toxcore/LAN_discovery.api.h >toxcore/LAN_discovery.h &
+"$APIDSL" toxcore/crypto_core.api.h >toxcore/crypto_core.h &
+"$APIDSL" toxcore/ping.api.h >toxcore/ping.h &
+"$APIDSL" toxcore/ping_array.api.h >toxcore/ping_array.h &
+"$APIDSL" toxcore/tox.api.h >toxcore/tox.h &
+"$APIDSL" toxav/toxav.api.h >toxav/toxav.h &
+"$APIDSL" toxencryptsave/toxencryptsave.api.h >toxencryptsave/toxencryptsave.h &
 set -x
 
-wait; wait; wait; wait; wait; wait; wait
+wait
+wait
+wait
+wait
+wait
+wait
+wait
 
 if grep '<unresolved>' ./*/*.h; then
   echo "error: some apidsl references were unresolved"
   exit 1
 fi
 
-CC_SOURCES=$(find . '(' -name '*.cc' ')')
-CC_SOURCES="$CC_SOURCES toxcore/crypto_core.c"
-CC_SOURCES="$CC_SOURCES toxcore/ping_array.c"
+readarray -t CC_SOURCES <<<"$(find . '(' -name '*.cc' ')')"
+CC_SOURCES+=(toxcore/crypto_core.c)
+CC_SOURCES+=(toxcore/ping_array.c)
 
 for bin in clang-format-6.0 clang-format-5.0 clang-format; do
   if which "$bin"; then
-    "$bin" -i -style='{BasedOnStyle: Google, ColumnLimit: 100}' $CC_SOURCES
+    "$bin" -i -style='{BasedOnStyle: Google, ColumnLimit: 100}' "${CC_SOURCES[@]}"
     break
   fi
 done
@@ -91,8 +100,8 @@ FIND="$FIND -and -not -wholename './super_donators/*'"
 FIND="$FIND -and -not -wholename './third_party/*'"
 FIND="$FIND -and -not -wholename './toxencryptsave/crypto_pwhash*'"
 
-C_SOURCES=$(eval "$FIND")
+readarray -t C_SOURCES <<<"$(eval "$FIND")"
 
-$ASTYLE -n --options=other/astyle/astylerc $C_SOURCES
+"$ASTYLE" -n --options=other/astyle/astylerc "${C_SOURCES[@]}"
 
-git diff --exit-code
+git diff --color=always --exit-code

other/docker/Dockerfile.ci

@@ -0,0 +1,75 @@
+# This Docker build emulates roughly what Travis CI is doing. It is not exactly
+# the same (different tool versions) and success in this image may not
+# necessarily mean success on Travis. This image is also not automatically
+# tested, so it may get out of date. Send PRs if you use it and it's broken.
+#
+# For one, we use bionic, not xenial, because xenial's clang is way too old.
+FROM ubuntu:16.04
+
+# Travis environment.
+RUN apt-get update && apt-get install --no-install-recommends -y \
+ apt-transport-https \
+ build-essential \
+ ca-certificates \
+ curl \
+ git \
+ pkg-config \
+ python-pip \
+ python-setuptools \
+ python3 \
+ software-properties-common \
+ wget \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+RUN curl https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \
+ && apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-6.0 main" \
+ && apt-get update && apt-get install --no-install-recommends -y \
+ clang-6.0 \
+ clang-format-6.0 \
+ llvm-6.0 \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+RUN ls /usr/bin/clang-6.0 && ln -s /usr/bin/clang-6.0 /usr/bin/clang \
+ && ls /usr/bin/clang++-6.0 && ln -s /usr/bin/clang++-6.0 /usr/bin/clang++ \
+ && ls /usr/bin/clang-format-6.0 && ln -s /usr/bin/clang-format-6.0 /usr/bin/clang-format \
+ && ls /usr/bin/opt-6.0 && ln -s /usr/bin/opt-6.0 /usr/bin/opt
+
+# Bionic's cmake is too old.
+RUN pip install --upgrade pip cmake
+
+# .travis.yml
+RUN apt-get update && apt-get install --no-install-recommends -y \
+ libconfig-dev \
+ libgtest-dev \
+ libopus-dev \
+ libsodium-dev \
+ libvpx-dev \
+ ninja-build \
+ pylint3 \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+# Set up travis user.
+RUN groupadd -r -g 1000 travis \
+ && useradd --no-log-init -r -g travis -u 1000 travis \
+ && mkdir -p /src/workspace /home/travis \
+ && chown travis:travis /home/travis
+USER travis
+
+# Set up environment.
+ENV CC=gcc CXX=g++ \
+PATH=/home/travis/.local/bin:$PATH \
+TRAVIS_REPO_SLUG=TokTok/c-toxcore
+
+# Copy minimal files to run "cmake-linux install", so we can avoid rebuilding
+# astyle and other things when only source files change.
+RUN mkdir -p /home/travis/build/c-toxcore /home/travis/cache
+WORKDIR /home/travis/build/c-toxcore
+COPY --chown=travis:travis c-toxcore/.travis/ /home/travis/build/c-toxcore/.travis/
+RUN .travis/cmake-linux install
+
+# Now copy the rest of the sources and run the build.
+COPY --chown=travis:travis . /home/travis/build/
+RUN .travis/cmake-linux script

other/docker/run-ci

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -eu
+
+readarray -t FILES <<<"$(git ls-files | sed -e 's,^,c-toxcore/,')"
+
+if [ -f .git ]; then
+  cd ..
+  tar -c "${FILES[@]}" "c-toxcore/.git" ".git/modules/c-toxcore" |
+    docker build -f c-toxcore/other/docker/Dockerfile.ci -
+else
+  cd ..
+  tar -c "${FILES[@]}" "c-toxcore/.git" |
+    docker build -f c-toxcore/other/docker/Dockerfile.ci -
+fi