Merge branch 'build-solving_docker_issues' into 'develop'

Build solving docker issues

See merge request !50

.gitlab-ci.yml

 stages:
-  - docker_login
+  - docker_pre_clean
   - ci
   - deploy
-  - docker_logout
+  - docker_post_clean
 
-docker_login:
-  stage: docker_login
+docker_pre_clean:
+  stage: docker_pre_clean
   script:
-    - docker login -u gitlab-ci-token -p ${CI_JOB_TOKEN} ${CI_REGISTRY}
+    - ${CI_PROJECT_DIR}/BuildServer/Docker/clean.sh
+  allow_failure: true
+  tags:
+    - docker
+    - sci
+    - ubuntu
+
+docker_post_clean:
+  stage: docker_post_clean
+  script:
+    - ${CI_PROJECT_DIR}/BuildServer/Docker/clean.sh
+  allow_failure: true
+  when: always
   tags:
     - docker
     - sci
@@ -19,8 +31,15 @@ docker_login:
 ci:ubuntu_trusty:
   stage: ci
   script:
-    - docker build --force-rm --build-arg MDANSE_GIT_COMMIT=${CI_COMMIT_SHA} -t ci_mdanse_trusty -f ${CI_PROJECT_DIR}/BuildServer/Docker/Builder/Ubuntu-trusty/Dockerfile_build .
+    - docker build --force-rm=true -t ci_mdanse_trusty -f ${CI_PROJECT_DIR}/BuildServer/Docker/Ubuntu-trusty/Dockerfile_setup .
+    - docker build --force-rm=true --build-arg CI_COMMIT_SHORT_SHA=${CI_COMMIT_SHORT_SHA} --build-arg CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME} --no-cache -t ci_mdanse_trusty_build -f ${CI_PROJECT_DIR}/BuildServer/Docker/Ubuntu-trusty/Dockerfile_build .
+    - docker run -iv${PWD}:/artifacts/ ci_mdanse_trusty_build
   allow_failure: false
+  artifacts:
+    paths:
+    - ${CI_PROJECT_DIR}/build
+    - ${CI_PROJECT_DIR}/temp
+    expire_in: 1 hrs
   tags:
     - docker
     - sci
@@ -29,9 +48,9 @@ ci:ubuntu_trusty:
 deploy:ubuntu_trusty:
   stage: deploy
   script:
-    - docker build --force-rm -t deploy_mdanse_trusty -f ${CI_PROJECT_DIR}/BuildServer/Docker/Builder/Ubuntu-trusty/Dockerfile_deploy .
-    - chmod 755 ${CI_PROJECT_DIR}/BuildServer/Docker/upload_artefacts.sh
-    - ${CI_PROJECT_DIR}/BuildServer/Docker/upload_artefacts.sh trusty
+    - docker build --force-rm=true -t ci_mdanse_trusty -f ${CI_PROJECT_DIR}/BuildServer/Docker/Ubuntu-trusty/Dockerfile_setup .
+    - docker build --force-rm=true --build-arg CI_COMMIT_SHORT_SHA=${CI_COMMIT_SHORT_SHA} --build-arg CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME} --no-cache -t ci_mdanse_trusty_build -f ${CI_PROJECT_DIR}/BuildServer/Docker/Ubuntu-trusty/Dockerfile_deploy .
+    - docker run -iv${PWD}:/artifacts/ ci_mdanse_trusty_build
   dependencies:
     - ci:ubuntu_trusty
   allow_failure: false
@@ -46,7 +65,7 @@ deploy:ubuntu_trusty:
   when: on_success
   artifacts:
     paths:
-    - ./*.deb
+    - ${CI_PROJECT_DIR}/*.deb
   tags:
     - docker
     - sci
@@ -58,8 +77,15 @@ deploy:ubuntu_trusty:
 ci:ubuntu_xenial:
   stage: ci
   script:
-    - docker build --force-rm --build-arg MDANSE_GIT_COMMIT=${CI_COMMIT_SHA} -t ci_mdanse_xenial -f ${CI_PROJECT_DIR}/BuildServer/Docker/Builder/Ubuntu-xenial/Dockerfile_build .
+    - docker build --force-rm=true -t ci_mdanse_xenial -f ${CI_PROJECT_DIR}/BuildServer/Docker/Ubuntu-xenial/Dockerfile_setup .
+    - docker build --force-rm=true --build-arg  CI_COMMIT_SHORT_SHA=${CI_COMMIT_SHORT_SHA} --build-arg CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME} --no-cache -t ci_mdanse_xenial_build -f ${CI_PROJECT_DIR}/BuildServer/Docker/Ubuntu-xenial/Dockerfile_build .
+    - docker run -iv${PWD}:/artifacts/ ci_mdanse_xenial_build
   allow_failure: false
+  artifacts:
+    paths:
+    - ${CI_PROJECT_DIR}/build
+    - ${CI_PROJECT_DIR}/temp
+    expire_in: 1 hrs
   tags:
     - docker
     - sci
@@ -68,9 +94,9 @@ ci:ubuntu_xenial:
 deploy:ubuntu_xenial:
   stage: deploy
   script:
-    - docker build --force-rm -t deploy_mdanse_xenial -f ${CI_PROJECT_DIR}/BuildServer/Docker/Builder/Ubuntu-xenial/Dockerfile_deploy .
-    - chmod 755 ${CI_PROJECT_DIR}/BuildServer/Docker/upload_artefacts.sh
-    - ${CI_PROJECT_DIR}/BuildServer/Docker/upload_artefacts.sh xenial
+    - docker build --force-rm=true -t ci_mdanse_xenial -f ${CI_PROJECT_DIR}/BuildServer/Docker/Ubuntu-xenial/Dockerfile_setup .
+    - docker build --force-rm=true --build-arg CI_COMMIT_SHORT_SHA=${CI_COMMIT_SHORT_SHA} --build-arg CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME} --no-cache -t ci_mdanse_xenial_build -f ${CI_PROJECT_DIR}/BuildServer/Docker/Ubuntu-xenial/Dockerfile_deploy .
+    - docker run -iv${PWD}:/artifacts/ ci_mdanse_xenial_build
   dependencies:
     - ci:ubuntu_xenial
   allow_failure: false
@@ -85,7 +111,7 @@ deploy:ubuntu_xenial:
   when: on_success
   artifacts:
     paths:
-    - ./*.deb
+    - ${CI_PROJECT_DIR}/*.deb
   tags:
     - docker
     - sci
@@ -97,8 +123,15 @@ deploy:ubuntu_xenial:
 ci:ubuntu_bionic:
   stage: ci
   script:
-    - docker build --force-rm --build-arg MDANSE_GIT_COMMIT=${CI_COMMIT_SHA} -t ci_mdanse_bionic -f ${CI_PROJECT_DIR}/BuildServer/Docker/Builder/Ubuntu-bionic/Dockerfile_build .
+    - docker build --force-rm=true -t ci_mdanse_bionic -f ${CI_PROJECT_DIR}/BuildServer/Docker/Ubuntu-bionic/Dockerfile_setup .
+    - docker build --force-rm=true --build-arg CI_COMMIT_SHORT_SHA=${CI_COMMIT_SHORT_SHA} --build-arg CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME} --build-arg CI_COMMIT_SHORT_SHA=${CI_COMMIT_SHORT_SHA} --build-arg CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME} --no-cache -t ci_mdanse_bionic_build -f ${CI_PROJECT_DIR}/BuildServer/Docker/Ubuntu-bionic/Dockerfile_build .
+    - docker run -iv${PWD}:/artifacts/ ci_mdanse_bionic_build
   allow_failure: false
+  artifacts:
+    paths:
+    - ${CI_PROJECT_DIR}/build
+    - ${CI_PROJECT_DIR}/temp
+    expire_in: 1 hrs
   tags:
     - docker
     - sci
@@ -107,9 +140,9 @@ ci:ubuntu_bionic:
 deploy:ubuntu_bionic:
   stage: deploy
   script:
-    - docker build --force-rm -t deploy_mdanse_bionic -f ${CI_PROJECT_DIR}/BuildServer/Docker/Builder/Ubuntu-bionic/Dockerfile_deploy .
-    - chmod 755 ${CI_PROJECT_DIR}/BuildServer/Docker/upload_artefacts.sh
-    - ${CI_PROJECT_DIR}/BuildServer/Docker/upload_artefacts.sh bionic
+    - docker build --force-rm=true -t ci_mdanse_bionic -f ${CI_PROJECT_DIR}/BuildServer/Docker/Ubuntu-bionic/Dockerfile_setup .
+    - docker build --force-rm=true --build-arg CI_COMMIT_SHORT_SHA=${CI_COMMIT_SHORT_SHA} --build-arg CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME} --no-cache -t ci_mdanse_bionic_build -f ${CI_PROJECT_DIR}/BuildServer/Docker/Ubuntu-bionic/Dockerfile_deploy .
+    - docker run -iv${PWD}:/artifacts/ ci_mdanse_bionic_build
   dependencies:
     - ci:ubuntu_bionic
   allow_failure: false
@@ -221,14 +254,3 @@ deploy:windows:
   tags:
     - sci
     - windows-10
-
-docker_logout:
-  stage: docker_logout
-  when: always
-  script:
-    - docker logout
-  tags:
-    - docker
-    - sci
-    - ubuntu
-

BuildServer/Docker/Builder/Ubuntu-bionic/Dockerfile_build

-FROM docker.ill.fr/scientific-software/mdanse:ubuntu-bionic
-
-SHELL ["/bin/bash", "-c"]
-
-ARG MDANSE_GIT_COMMIT
-
-RUN GIT_SSL_NO_VERIFY=true git clone https://code.ill.fr/scientific-software/mdanse.git
-
-WORKDIR mdanse
-
-RUN git checkout ${MDANSE_GIT_COMMIT}
-
-RUN source ./BuildServer/Unix/Debian/definitions.sh; source ./BuildServer/Unix/setup_ci.sh ; ./BuildServer/Unix/build.sh
-
-RUN source ./BuildServer/Unix/Debian/definitions.sh; source ./BuildServer/Unix/setup_ci.sh ; ./BuildServer/Unix/tests.sh
-

BuildServer/Docker/Builder/Ubuntu-bionic/Dockerfile_deploy

-FROM ci_mdanse_bionic
-
-SHELL ["/bin/bash", "-c"]
-
-VOLUME builds
-
-RUN source ./BuildServer/Unix/Debian/definitions.sh; source ./BuildServer/Unix/setup_ci.sh ; source ./BuildServer/Unix/bionic_dependencies_modification.sh ; ./BuildServer/Unix/Debian/deploy.sh
-
-

BuildServer/Docker/Builder/Ubuntu-trusty/Dockerfile_build

-FROM docker.ill.fr/scientific-software/mdanse:ubuntu-trusty
-
-SHELL ["/bin/bash", "-c"]
-
-ARG MDANSE_GIT_COMMIT
-
-RUN GIT_SSL_NO_VERIFY=true git clone https://code.ill.fr/scientific-software/mdanse.git
-
-WORKDIR mdanse
-
-RUN git checkout ${MDANSE_GIT_COMMIT}
-
-RUN source ./BuildServer/Unix/Debian/definitions.sh; source ./BuildServer/Unix/setup_ci.sh ; ./BuildServer/Unix/build.sh
-
-RUN source ./BuildServer/Unix/Debian/definitions.sh; source ./BuildServer/Unix/setup_ci.sh ; ./BuildServer/Unix/tests.sh
-

BuildServer/Docker/Builder/Ubuntu-trusty/Dockerfile_deploy

-FROM ci_mdanse_trusty
-
-SHELL ["/bin/bash", "-c"]
-
-VOLUME builds
-
-RUN source ./BuildServer/Unix/Debian/definitions.sh; source ./BuildServer/Unix/setup_ci.sh ; ./BuildServer/Unix/Debian/deploy.sh
-
-

BuildServer/Docker/Builder/Ubuntu-xenial/Dockerfile_build

-FROM docker.ill.fr/scientific-software/mdanse:ubuntu-xenial
-
-SHELL ["/bin/bash", "-c"]
-
-ARG MDANSE_GIT_COMMIT
-
-RUN GIT_SSL_NO_VERIFY=true git clone https://code.ill.fr/scientific-software/mdanse.git
-
-WORKDIR mdanse
-
-RUN git checkout ${MDANSE_GIT_COMMIT}
-
-RUN source ./BuildServer/Unix/Debian/definitions.sh; source ./BuildServer/Unix/setup_ci.sh ; ./BuildServer/Unix/build.sh
-
-RUN source ./BuildServer/Unix/Debian/definitions.sh; source ./BuildServer/Unix/setup_ci.sh ; ./BuildServer/Unix/tests.sh
-

BuildServer/Docker/Builder/Ubuntu-xenial/Dockerfile_deploy

-FROM ci_mdanse_xenial
-
-SHELL ["/bin/bash", "-c"]
-
-VOLUME builds
-
-RUN source ./BuildServer/Unix/Debian/definitions.sh; source ./BuildServer/Unix/setup_ci.sh ; ./BuildServer/Unix/Debian/deploy.sh
-
-

BuildServer/Docker/Setup/build.sh

-#!/bin/bash
-
-docker build --force-rm -t docker.ill.fr/scientific-software/mdanse:$1 -f $2/Dockerfile .
-

BuildServer/Docker/Ubuntu-bionic/Dockerfile_build

+FROM ci_mdanse_bionic
+
+SHELL ["/bin/bash", "-c"]
+
+ARG CI_COMMIT_SHORT_SHA
+ARG CI_COMMIT_REF_NAME
+
+ENV CI_COMMIT_SHORT_SHA=${CI_COMMIT_SHORT_SHA}
+ENV CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME}
+
+COPY ./ /app/
+WORKDIR /app
+
+CMD source ./BuildServer/Unix/Debian/definitions.sh;  source ./BuildServer/Unix/setup_ci.sh; ./BuildServer/Unix/build.sh; ./BuildServer/Unix/tests.sh; cp -a ${CI_PROJECT_DIR}/build /artifacts/; cp -a ${CI_PROJECT_DIR}/temp /artifacts/
\ No newline at end of file

BuildServer/Docker/Ubuntu-bionic/Dockerfile_deploy

+FROM ci_mdanse_bionic
+
+SHELL ["/bin/bash", "-c"]
+
+ARG CI_COMMIT_SHORT_SHA
+ARG CI_COMMIT_REF_NAME
+
+ENV CI_COMMIT_SHORT_SHA=${CI_COMMIT_SHORT_SHA}
+ENV CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME}
+
+COPY ./ /app/
+WORKDIR /app
+
+CMD source ./BuildServer/Unix/Debian/definitions.sh;  source ./BuildServer/Unix/setup_ci.sh; source ./BuildServer/Unix/bionic_dependencies_modification.sh; ./BuildServer/Unix/Debian/deploy.sh; cp -a *.deb /artifacts/
\ No newline at end of file

BuildServer/Docker/Setup/Ubuntu-bionic/Dockerfile → BuildServer/Docker/Ubuntu-bionic/Dockerfile_setup

 FROM ubuntu:bionic
 
+ENV https_proxy http://proxy.ill.fr:8888
+ENV http_proxy http://proxy.ill.fr:8888
+
 RUN export DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update
@@ -26,5 +29,3 @@ RUN apt-get install -y --no-install-recommends python-sphinx
 RUN apt-get install -y --no-install-recommends pyro
 
 RUN apt-get install -y --no-install-recommends lsb-release
\ No newline at end of file
-
-

BuildServer/Docker/Ubuntu-trusty/Dockerfile_build

+FROM ci_mdanse_trusty
+
+SHELL ["/bin/bash", "-c"]
+
+ARG CI_COMMIT_SHORT_SHA
+ARG CI_COMMIT_REF_NAME
+
+ENV CI_COMMIT_SHORT_SHA=${CI_COMMIT_SHORT_SHA}
+ENV CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME}
+
+COPY ./ /app/
+WORKDIR /app
+
+CMD source ./BuildServer/Unix/Debian/definitions.sh;  source ./BuildServer/Unix/setup_ci.sh; ./BuildServer/Unix/build.sh; ./BuildServer/Unix/tests.sh; cp -a ${CI_PROJECT_DIR}/build /artifacts/; cp -a ${CI_PROJECT_DIR}/temp /artifacts/
\ No newline at end of file

BuildServer/Docker/Ubuntu-trusty/Dockerfile_deploy

+FROM ci_mdanse_trusty
+
+SHELL ["/bin/bash", "-c"]
+
+ARG CI_COMMIT_SHORT_SHA
+ARG CI_COMMIT_REF_NAME
+
+ENV CI_COMMIT_SHORT_SHA=${CI_COMMIT_SHORT_SHA}
+ENV CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME}
+
+COPY ./ /app/
+WORKDIR /app
+
+CMD source ./BuildServer/Unix/Debian/definitions.sh;  source ./BuildServer/Unix/setup_ci.sh; ./BuildServer/Unix/Debian/deploy.sh; cp -a *.deb /artifacts/
\ No newline at end of file

BuildServer/Docker/Setup/Ubuntu-trusty/Dockerfile → BuildServer/Docker/Ubuntu-trusty/Dockerfile_setup

 FROM ubuntu:trusty
 
+ENV https_proxy http://proxy.ill.fr:8888
+ENV http_proxy http://proxy.ill.fr:8888
+
+RUN export DEBIAN_FRONTEND=noninteractive
+
 RUN apt-get update
 
 RUN apt-get install -y tzdata
@@ -24,4 +29,3 @@ RUN apt-get install -y --no-install-recommends python-sphinx
 RUN apt-get install -y --no-install-recommends pyro
 
 RUN apt-get install -y --no-install-recommends lsb-release
\ No newline at end of file
-

BuildServer/Docker/Ubuntu-xenial/Dockerfile_build

+FROM ci_mdanse_xenial
+
+SHELL ["/bin/bash", "-c"]
+
+ARG CI_COMMIT_SHORT_SHA
+ARG CI_COMMIT_REF_NAME
+
+ENV CI_COMMIT_SHORT_SHA=${CI_COMMIT_SHORT_SHA}
+ENV CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME}
+
+COPY ./ /app/
+WORKDIR /app
+
+CMD source ./BuildServer/Unix/Debian/definitions.sh;  source ./BuildServer/Unix/setup_ci.sh; ./BuildServer/Unix/build.sh; ./BuildServer/Unix/tests.sh; cp -a ${CI_PROJECT_DIR}/build /artifacts/; cp -a ${CI_PROJECT_DIR}/temp /artifacts/
\ No newline at end of file

BuildServer/Docker/Ubuntu-xenial/Dockerfile_deploy

+FROM ci_mdanse_xenial
+
+SHELL ["/bin/bash", "-c"]
+
+ARG CI_COMMIT_SHORT_SHA
+ARG CI_COMMIT_REF_NAME
+
+ENV CI_COMMIT_SHORT_SHA=${CI_COMMIT_SHORT_SHA}
+ENV CI_COMMIT_REF_NAME=${CI_COMMIT_REF_NAME}
+
+COPY ./ /app/
+WORKDIR /app
+
+CMD source ./BuildServer/Unix/Debian/definitions.sh;  source ./BuildServer/Unix/setup_ci.sh; ./BuildServer/Unix/Debian/deploy.sh; cp -a *.deb /artifacts/
\ No newline at end of file

BuildServer/Docker/Setup/Ubuntu-xenial/Dockerfile → BuildServer/Docker/Ubuntu-xenial/Dockerfile_setup

 FROM ubuntu:xenial
 
+ENV https_proxy http://proxy.ill.fr:8888
+ENV http_proxy http://proxy.ill.fr:8888
+
+RUN export DEBIAN_FRONTEND=noninteractive
+
 RUN apt-get update
 
 RUN apt-get install -y tzdata
@@ -24,4 +29,3 @@ RUN apt-get install -y --no-install-recommends python-sphinx
 RUN apt-get install -y --no-install-recommends pyro
 
 RUN apt-get install -y --no-install-recommends lsb-release
\ No newline at end of file
-

BuildServer/Docker/clean.sh

 # Return the docker containers that are exited
 exited_containers=`docker ps -q -f "status=exited"`
-
 # If some have been found, remove them
 if [ -n "$exited_containers" ]
 then
     docker rm -f ${exited_containers}
 fi
 
-# Return the docker images corresponding to MDANSE
-images=`docker images -a --format "{{.Repository}}:{{.Tag}}" | grep _mdanse_` 
-
-# If some have been found, remove them
-if [ -n "$images" ]
-then
-    docker rmi -f ${images}
-fi
-
 # Return the docker images that are in dangling state
 images=`docker images -q -f "dangling=true"`
-
 # If some dangling images have been found, remove them
 if [ -n "$images" ]
 then
     docker rmi -f ${images}
 fi
-

BuildServer/Unix/setup_ci.sh

@@ -17,20 +17,22 @@ mkdir -p ${CI_TEMP_DIR}
 
 cd ${CI_PROJECT_DIR}
 
-# Get revision number from Git
-if [ -z ${CI_COMMIT_SHA} ]; then
-    export CI_COMMIT_ID=$(git rev-parse HEAD)
-else
-    export CI_COMMIT_ID=${CI_COMMIT_SHA}
-fi
-export CI_COMMIT_ID=${CI_COMMIT_ID:0:8}
-
-# Get commit branch from Gitlab
-if [ -z ${CI_COMMIT_REF_NAME} ]; then
-    CI_COMMIT_REF_NAME=$(git show -s --pretty=%d HEAD)
-    CI_COMMIT_REF_NAME=$(echo ${CI_COMMIT_REF_NAME} | rev | cut -d, -f1 | cut -c2- | cut -d/ -f1 | rev)
-    export CI_COMMIT_REF_NAME
-fi
+## Get revision number from Git
+CI_COMMIT_ID=${CI_COMMIT_SHORT_SHA}
+#if [ -z ${CI_COMMIT_SHA} ]; then
+#    export CI_COMMIT_ID=$(git rev-parse HEAD)
+#else
+#    export CI_COMMIT_ID=${CI_COMMIT_SHA}
+#fi
+#export CI_COMMIT_ID=${CI_COMMIT_ID:0:8}
+#
+## Get commit branch from Gitlab
+#if [ -z ${CI_COMMIT_REF_NAME} ]; then
+#    CI_COMMIT_REF_NAME=$(git show -s --pretty=%d HEAD)
+#    CI_COMMIT_REF_NAME=$(echo ${CI_COMMIT_REF_NAME} | rev | cut -d, -f1 | cut -c2- | cut -d/ -f1 | rev)
+#    export CI_COMMIT_REF_NAME
+#fi
+#
 
 echo -e "${BLUE}""Commit id = ${CI_COMMIT_ID}""${NORMAL}"
 echo -e "${BLUE}""Branch name = ${CI_COMMIT_REF_NAME}""${NORMAL}"

BuildServer/Windows/definitions.bat

@@ -8,20 +8,3 @@ if "%BUILD_TARGET%"=="win32" (
 	set BUILD_TARGET=win-amd64
 	set MSVC_BUILD_TARGET=x64
 )
\ No newline at end of file
-
-rem Set the source directory
-set MDANSE_SOURCE_DIR=%cd%
-
-rem Set th directory where the MDANSE dependencies are stored
-set MDANSE_DEPENDENCIES_DIR=C:\\Projects\\mdanse\\resources\\dependencies\\%BUILD_TARGET%
-
-rem Set the location where the temporary Python will be installed
-set MDANSE_TEMPORARY_INSTALLATION_DIR=%MDANSE_SOURCE_DIR%\\BuildServer\\Windows\\Build
-
-rem Get revision number from Git
-rem To understand this syntax "set cmd=...... for /F %%ii ......", see https://stackoverflow.com/questions/2323292/assign-output-of-a-program-to-a-variable
-set cmd="git rev-parse --short HEAD"
-for /F %%i in (' %cmd% ') do set MDANSE_GIT_CURRENT_COMMIT=%%i
-
-rem Get commit branch from Gitlab
-set MDANSE_GIT_BRANCH_NAME=%CI_COMMIT_REF_NAME%