From 127096e7f8811540a956af6085d62dd587d27778 Mon Sep 17 00:00:00 2001 From: Mouginot Baptiste Date: Mon, 29 Aug 2016 12:07:39 -0500 Subject: [PATCH] [Rdy for review] Debian package generation (#417) * first add in ci-testintg branch * rename docker folder to master... * update symb link * update CMakelist * building organize folder * first version of the docker file for ubuntu deb generation * update cirecle.yml * comment nosetest for speed testeing * change workdir after git clone but BEFORE fetching.. * add ls -alh to see the sixy of the deb, add also a timeout of 20minutes * correction on the time out inplementation * cleaning un-necessary stuff * it seems to be working on deb_generation branch, so I move the debian package generation to the master branch * first use of the script * adding the script... * fix branch name in circle.yml and remove bad : in build script * remane Docker file in release * adding the file... * add wget on the list of package installed * correct Docker_sample * changing the way to recover hash of the commit, and to add it in the .deb name * rework docker/README * add stable explicit call in DockerFile * change XX and HASH template variables to CYCAMORE_UBUNTU_MAJOR_VERSION, CYCAMORE_REPO_HASH, add a tmp Dockerfile for the second sed to work... * form differently the deb package name * remove explicit call to docker cyclus:stable container, it is not to be found... * _/- swap * _/- swap * updating deb to proper name * bad deb name fix * Working, putting those change on master branch trigger * uncommenting nosetest * old files, rename script * rename Docker_sample in Docker_template * correct typo * symb link should point to the dev version by default * change the ci-trigger to deb_generation branch * revert Dockerfile to old version * remaning release-ci folder into deb-ci * update Dockerfile (comments+installation destination) * modify build script accordingly to cyclus one * update Docker README * change folder name in circle file * change folder name in script * trying to fix the isntallation of cycamore headers in cyclus include dir * revert stupid commit... * trying to build cycamore docker against cyclus/cyclus:stable * trying to force latest to see if the issu stay.. * moving back master/docker to cyclus:stable,as master:stable should be push with the cyclus release * correct dev into deb * cutting CMakeList line * wrong branch name... * working back on master trigger * set trigger on deb_generation to ensure deb still working * working; back on master trigger * switch trigger to ci-testing, remove discovered dependencies option in CPACK (might reduce ubuntu 16.04 dependency lists * add cyclus into the dependency list * correct typo * back on master --- CMakeLists.txt | 87 +++++++++++++++----- Dockerfile | 2 +- circle.yml | 17 ++++ docker/README.md | 34 ++++++-- docker/deb-ci/Dockerfile_template | 43 ++++++++++ docker/deb-ci/build_upload_deb.sh | 16 ++++ docker/develop-ci/Dockerfile | 9 ++ docker/{cycamore-ci => master-ci}/Dockerfile | 2 +- 8 files changed, 180 insertions(+), 30 deletions(-) create mode 100644 docker/deb-ci/Dockerfile_template create mode 100755 docker/deb-ci/build_upload_deb.sh create mode 100644 docker/develop-ci/Dockerfile rename docker/{cycamore-ci => master-ci}/Dockerfile (85%) diff --git a/CMakeLists.txt b/CMakeLists.txt index fdc93cbefe..dc11099a6c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -230,32 +230,90 @@ IF(NOT CYCLUS_DOC_ONLY) # COMPONENT data # ) - # Here we set some components for installation with cpack - SET(CPACK_COMPONENTS_ALL cycamore testing libraries data core) + ############################################################################################## + ################################## begin uninstall target #################################### + ############################################################################################## + + CONFIGURE_FILE( + "${CMAKE_CURRENT_SOURCE_DIR}/config/cmake_uninstall.cmake.in" + "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake" + IMMEDIATE @ONLY + ) + + ADD_CUSTOM_TARGET(uninstall + COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake) + + ############################################################################################## + ################################### end uninstall target ##################################### + ############################################################################################## + + ############################################################################################## + ####################################### begin cpack ########################################## + ############################################################################################## + + # Here's where we package it with CPack SET(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Cycamore : The Cyclus Additional Module Repository.") + + # Here we set some components for installation with cpack + SET(CPACK_COMPONENTS_ALL cycamore testing libraries data core) + SET(CPACK_GENERATOR "DEB") + + # Organisation SET(CPACK_DEBIAN_PACKAGE_MAINTAINER "CNERG, UW-Madison") #required SET(CPACK_PACKAGE_VENDOR "CNERG, UW-Madison") SET(CPACK_PACKAGE_DESCRIPTION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/README.rst") SET(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.rst") + + # Version SET(CPACK_PACKAGE_VERSION_MAJOR "1") - SET(CPACK_PACKAGE_VERSION_MINOR "0") - #SET(CPACK_PACKAGE_VERSION_PATCH "0") # should use commit number here ? - SET(CPACK_PACKAGE_INSTALL_DIRECTORY "cycamore${CPACK_VERSION_MAJOR}.${CPACK_VERSION_MINOR}") - SET(CPACK_PACKAGE_NAME "${CMAKE_PROJECT_NAME}") - - CONFIGURE_FILE("${CYCAMORE_SOURCE_DIR}/cmake/CycamoreCPackOptions.cmake.in" - "${CYCAMORE_BINARY_DIR}/cmake/CycamoreCPackOptions.cmake" @ONLY) + SET(CPACK_PACKAGE_VERSION_MINOR "4") + SET(CPACK_PACKAGE_VERSION_PATCH "0") + + # Dependencies + SET(CPACK_DEBIAN_PACKAGE_DEPENDS "cyclus (>= 1.4.0)") + SET(CPACK_DEBIAN_PACKAGE_DEPENDS "${CPACK_DEBIAN_PACKAGE_DEPENDS}, libtcmalloc-minimal4 (>= 2.1)") + SET(CPACK_DEBIAN_PACKAGE_DEPENDS "${CPACK_DEBIAN_PACKAGE_DEPENDS}, libboost-filesystem-dev (>= 1.54.0)") + SET(CPACK_DEBIAN_PACKAGE_DEPENDS "${CPACK_DEBIAN_PACKAGE_DEPENDS}, libboost-program-options-dev (>= 1.54.0)") + SET(CPACK_DEBIAN_PACKAGE_DEPENDS "${CPACK_DEBIAN_PACKAGE_DEPENDS}, libboost-serialization-dev (>= 1.54.0)") + SET(CPACK_DEBIAN_PACKAGE_DEPENDS "${CPACK_DEBIAN_PACKAGE_DEPENDS}, libhdf5-dev (>= 1.8.11)") + SET(CPACK_DEBIAN_PACKAGE_DEPENDS "${CPACK_DEBIAN_PACKAGE_DEPENDS}, libxml++2.6-dev (>= 2.36.0)") + SET(CPACK_DEBIAN_PACKAGE_DEPENDS "${CPACK_DEBIAN_PACKAGE_DEPENDS}, coinor-libcbc-dev (>= 2.8.7)") + MESSAGE("CPACK_DEBIAN_PACKAGE_DEPENDS ${CPACK_DEBIAN_PACKAGE_DEPENDS}") + + # Names + SET(CPACK_PACKAGE_NAME "cycamore") + SET(CPACK_PACKAGE_INSTALL_DIRECTORY "cycamore_${CPACK_PACKAGE_VERSION_MAJOR}") + SET(CPACK_PACKAGE_INSTALL_DIRECTORY "${CPACK_PACKAGE_INSTALL_DIRECTORY}.${CPACK_PACKAGE_VERSION_MINOR}") + SET(CPACK_PACKAGE_INSTALL_DIRECTORY "${CPACK_PACKAGE_INSTALL_DIRECTORY}.${CPACK_PACKAGE_VERSION_PATCH}") + + SET(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}") + SET(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_FILE_NAME}_${CPACK_PACKAGE_VERSION_MAJOR}") + SET(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_FILE_NAME}.${CPACK_PACKAGE_VERSION_MINOR}") + SET(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_FILE_NAME}.${CPACK_PACKAGE_VERSION_PATCH}") + SET(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_FILE_NAME}_${CPACK_HASH_NAME}") + SET(CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_FILE_NAME}_${CPACK_SYSTEM_NAME}") + + # Configuration + SET(CPACK_INCLUDE_TOPLEVEL_DIRECTORY "1") + SET(CPACK_DEBIAN_PACKAGE_SHLIBDEPS OFF) + CONFIGURE_FILE( + "${CYCAMORE_SOURCE_DIR}/cmake/CycamoreCPackOptions.cmake.in" + "${CYCAMORE_BINARY_DIR}/cmake/CycamoreCPackOptions.cmake" @ONLY + ) SET(CPACK_PROJECT_CONFIG_FILE "${CYCAMORE_BINARY_DIR}/cmake/CycamoreCPackOptions.cmake") SET(CPACK_PACKAGE_EXECUTABLES "cycamore" "cycamore_unit_tests") - INCLUDE(CPack) SET(ROOT_DIR ${CYCLUS_ROOT_DIR}) SET(PROJ_DIR ${PROJECT_BINARY_DIR}) SET(CORE_SHARE_DIR ${CYCLUS_CORE_SHARE_DIR}) + + ############################################################################################## + ######################################## end cpack ########################################### + ############################################################################################## ENDIF(NOT CYCLUS_DOC_ONLY) @@ -270,12 +328,3 @@ IF(DOXYGEN_FOUND) ELSE(DOXYGEN_FOUND) MESSAGE(STATUS "WARNING: Doxygen not found - doc won't be created") ENDIF(DOXYGEN_FOUND) - -# uninstall target -configure_file( - "${CMAKE_CURRENT_SOURCE_DIR}/config/cmake_uninstall.cmake.in" - "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake" - IMMEDIATE @ONLY) - -add_custom_target(uninstall - COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake) diff --git a/Dockerfile b/Dockerfile index b6f63b7756..59d9ea527c 120000 --- a/Dockerfile +++ b/Dockerfile @@ -1 +1 @@ -docker/cycamore-ci/Dockerfile \ No newline at end of file +docker/develop-ci/Dockerfile \ No newline at end of file diff --git a/circle.yml b/circle.yml index 39b7862256..6bed7dc280 100644 --- a/circle.yml +++ b/circle.yml @@ -17,3 +17,20 @@ deployment: commands: - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS - docker push cyclus/cycamore:latest + + master: + branch: master + commands: + - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS + # Update cyclus/cylus:stable docker container + - rm Dockerfile && ln -s docker/master-ci/Dockerfile # place the correct Dockerfile + - docker build -t cyclus/cycamore:stable . + - docker push cyclus/cycamore:stable + # DEB GENERATION + #ubuntu 14.04 + - docker/deb-ci/build_upload_deb.sh 14 : + timeout: 1200 # set 20 minute timeout (to avoid CI to fail) + # Ubuntu 16.04 + - docker/deb-ci/build_upload_deb.sh 16 : + timeout: 1200 # set 20 minute timeout (to avoid CI to fail) + ## END DEB GENERATION diff --git a/docker/README.md b/docker/README.md index de2a9aea99..504c3a582e 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,13 +1,29 @@ -Each subdirectory contains a dockerfile that does something useful: - -* ``cycamore-ci`` is the dockerfile used for running cycamore on a continuous - integration service. This dockerfile assumes that the current working - directory is a cycamore repository - and that version of cycamore is copied - into the docker container and used for the build. The dockerfile in the - cycamore repository root is a symbolic link to this dockerfile. This - dockerfile uses the base image ``cyclus/cyclus`` from the docker hub +Each subdirectory contains a Dockerfile that does something useful: + +* ``develop-ci`` is the Dockerfile used for running Cycamore on a continuous + integration service. This Dockerfile assumes that the current working + directory is a Cycamore repository - and that version of Cycamore is copied + into the Docker container and used for the build. The Dockerfile in the + Cycamore repository root is a symbolic link to this Dockerfile. This + Dockerfile uses the base image ``cyclus/cyclus:latest`` from the Docker hub + repository. This Docker container is uploaded as ``cyclus/Cycamore:latest`` on + the Docker hub when merging a PR on the `develop` branch of the Cycamore Github repository. -* ``cloudlus-worker`` +* ``master-ci`` is the Dockerfile used for running last stable Cycamore version + a continuous integration service. This Dockerfile works exactly as the + develop-ci one, except that it builds Cycamore against the + ``cyclus/cyclus:stable`` from the Docker hub repository. This Docker container + is also uploaded on the Docker hub as the ``cyclus/Cycamore:stable`` when + merging a PR on the `master` branch of Cycamore Github repository. + +* ``deb-ci`` contains a script which allows to build the Dockerfile require + to generate a Cycamore Debian package for the installation of Cycamore on + ubuntu. The script take the major version number of ubuntu as a parameter. It + builds Cycamore on the corresponding Ubuntu-LTS version, after installing all + Cycamore dependencies -including Cyclus- and generates the Cycamore Debian + installation package that need to be extracted from the Docker container + (tested on Ubuntu 14.04 & 16.04) + diff --git a/docker/deb-ci/Dockerfile_template b/docker/deb-ci/Dockerfile_template new file mode 100644 index 0000000000..da9b5f6714 --- /dev/null +++ b/docker/deb-ci/Dockerfile_template @@ -0,0 +1,43 @@ +FROM ubuntu:CYCAMORE_UBUNTU_MAJOR_VERSION.04 + +# Because we used specific Ubuntu version can't use cyclus-deps docker container +RUN apt-get -y --force-yes update + +RUN apt-get install -y --force-yes \ + cmake \ + make \ + libboost-all-dev \ + libxml2-dev \ + libxml++2.6-dev \ + libsqlite3-dev \ + libhdf5-serial-dev \ + libbz2-dev \ + coinor-libcbc-dev \ + coinor-libcoinutils-dev \ + coinor-libosi-dev \ + coinor-libclp-dev \ + coinor-libcgl-dev \ + libblas-dev \ + liblapack-dev \ + g++ \ + libgoogle-perftools-dev \ + git \ + python \ + python-tables \ + python-numpy \ + python-nose \ + wget + +RUN wget -r --no-parent -A 'cyclus_1.4.0_*_ubuntu.CYCAMORE_UBUNTU_MAJOR_VERSION.04.deb' http://dory.fuelcycle.org:4848 +run mv dory.fuelcycle.org:4848/* ./; rm -rf dory.fuelcycle.org:4848 +RUN dpkg -i cyclus_1.4.0_*_ubuntu.CYCAMORE_UBUNTU_MAJOR_VERSION.04.deb + +COPY . /cycamore +WORKDIR cycamore +RUN mkdir -p deb +WORKDIR /cycamore/deb +RUN cmake -DCPACK_PACKAGING_INSTALL_PREFIX=/usr/local -DCMAKE_INSTALL_PREFIX=/usr/local -DCPACK_HASH_NAME=CYCAMORE_REPO_HASH -DCPACK_SYSTEM_NAME=ubuntu.CYCAMORE_UBUNTU_MAJOR_VERSION.04 .. + +# Build the deb file in the docker container +RUN cpack -v .. + diff --git a/docker/deb-ci/build_upload_deb.sh b/docker/deb-ci/build_upload_deb.sh new file mode 100755 index 0000000000..14960db441 --- /dev/null +++ b/docker/deb-ci/build_upload_deb.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +if [ $# -ne 1 ]; then + echo "illegal number of parameters" + exit 1 +fi + +rm Dockerfile +sed -e s/CYCAMORE_UBUNTU_MAJOR_VERSION/${1}/g docker/deb-ci/Dockerfile_template > Dockerfile +sed -i s/CYCAMORE_REPO_HASH/`git rev-parse --short HEAD`/g Dockerfile + +docker build -t deb/ubuntu-${1} . # build cyclus against ubuntu 14.04 in the docker container +docker create --name=deb-${1}.04 deb/ubuntu-${1} # build deb in docker +docker cp deb-${1}.04:/cycamore/deb ./ # extract deb +scp ./deb/cycamore_*_ubuntu.${1}.04.deb root@dory.fuelcycle.org:./cyclus-deb/ # push the deb on dory +rm -rf deb # clean the deb folder diff --git a/docker/develop-ci/Dockerfile b/docker/develop-ci/Dockerfile new file mode 100644 index 0000000000..acd3302f96 --- /dev/null +++ b/docker/develop-ci/Dockerfile @@ -0,0 +1,9 @@ +FROM cyclus/cyclus:latest + +COPY . /cycamore +WORKDIR /cycamore +RUN mkdir Release +WORKDIR /cycamore/Release +RUN cmake .. -DCMAKE_BUILD_TYPE=Release +RUN make -j2 && make install + diff --git a/docker/cycamore-ci/Dockerfile b/docker/master-ci/Dockerfile similarity index 85% rename from docker/cycamore-ci/Dockerfile rename to docker/master-ci/Dockerfile index abf25deb71..f53c240567 100644 --- a/docker/cycamore-ci/Dockerfile +++ b/docker/master-ci/Dockerfile @@ -1,5 +1,5 @@ -FROM cyclus/cyclus +FROM cyclus/cyclus:stable COPY . /cycamore WORKDIR /cycamore