Refactoring of the BuildServer scripts, adding the beta indication in hte GUI

.gitignore

-/.project
-/.pydevproject
-/build/
-*.pyc
+.DS_Store
 .env
 .eggs
-dist/
-.DS_Store
-Tests/FunctionalTests/Jobs
-/.gitignore
-/MANIFEST
+.gitignore
+.project
+.pydevproject
 .settings
+MANIFEST
+*.pyc
+build/
+dist/
+

.gitlab-ci.yml

 stages:
   - ci
   - deploy
+  - clean
  
-# trusty pipeline
-ci:trusty:
+# xenial pipeline
+ci:xenial:
   stage: ci
   script:
-    - ./BuildServer/Debian/run_build.sh
-    - ./BuildServer/Debian/run_tests.sh
+    - source ./BuildServer/Unix/definitions_debian.sh
+    - source ./BuildServer/Unix/version.sh
+    - ./BuildServer/Unix/clean.sh
+    - ./BuildServer/Unix/build.sh
+    - ./BuildServer/Unix/tests.sh
   allow_failure: false
   only:
     - triggers
@@ -19,33 +23,39 @@ ci:trusty:
     - /^bugfix-.*$/
   artifacts:
     paths:
-    - build
+    - ./BuildServer/Unix/Build
     expire_in: 1 day
   tags:
-    - ubuntu16.04
-
-deploy:trusty:
+    - ubuntu14.04
+deploy:xenial:
   stage: deploy
   script:
-    - ./BuildServer/Debian/run_deploy.sh amd64 trusty
+    - ./BuildServer/Unix/deploy_debian.sh
   only:
     - tags
     - develop
   dependencies:
-    - ci:trusty
+    - ci:xenial
   when: on_success
   artifacts:
     paths:
-    - ./BuildServer/Debian/Build/*.deb
+     - BuildServer/MDANSE*.deb
   tags:
-    - ubuntu16.04
+    - ubuntu14.04
+clean:xenial:
+  stage: clean
+  script:
+    - ./BuildServer/Unix/clean.sh
 
 # osx pipeline
 ci:osx:
   stage: ci
   script:
-    - ./BuildServer/Darwin/Scripts/run_build.sh
-    - ./BuildServer/Darwin/Scripts/run_tests.sh
+    - source ./BuildServer/Unix/definitions_macos.sh
+    - source ./BuildServer/Unix/version.sh
+    - ./BuildServer/Unix/clean.sh
+    - ./BuildServer/Unix/build.sh
+    - ./BuildServer/Unix/tests.sh
   allow_failure: false
   only:
     - triggers
@@ -57,33 +67,14 @@ ci:osx:
     - /^bugfix-.*$/
   artifacts:
     paths:
-    - build
+    - ./BuildServer/Unix/Build
     expire_in: 1 day
   tags:
-    - macos10.13
-
-test:osx:
-  stage: test
-  script:
-    - ./BuildServer/Darwin/Scripts/run_tests.sh
-  allow_failure: false
-  only:
-    - triggers
-    - tags
-    - release
-    - hotfix
-    - develop
-    - /^feature-.*$/
-    - /^bugfix-.*$/
-  dependencies:
-    - build:osx
-  tags:
-    - macos10.13
-
+    - macmini2
 deploy:osx:
   stage: deploy
   script:
-    - ./BuildServer/Darwin/Scripts/run_deploy.sh
+    - ./BuildServer/Unix/deploy_macos.sh
   only:
     - tags
     - develop
@@ -92,16 +83,24 @@ deploy:osx:
   when: on_success
   artifacts:
     paths:
-     - BuildServer/Darwin/Build/*.dmg
+     - BuildServer/MDANSE*.dmg
   tags:
-    - macos10.13
+    - macmini2
+clean:osx:
+  stage: clean
+  script:
+    - ./BuildServer/Unix/clean.sh
 
 # windows 7 pipeline
+# "call" is mandatory, see https://gitlab.com/gitlab-org/gitlab-runner/issues/1051
 ci:windows:
   stage: ci
   script:
-    - .\BuildServer\\Windows\\run_build.bat
-    - .\BuildServer\\Windows\\run_tests.bat
+    - call .\\BuildServer\\Windows\\definitions.bat
+    - call .\\BuildServer\\Windows\\version.bat
+    - call .\\BuildServer\\Windows\\clean.bat
+    - call .\\BuildServer\\Windows\\build.bat
+    - call .\\BuildServer\\Windows\\tests.bat
   allow_failure: false
   only:
     - triggers
@@ -113,17 +112,14 @@ ci:windows:
     - /^bugfix-.*$/
   artifacts:
     paths:
-    - .\BuildServer\Windows\Build
+    - call .\\BuildServer\\Windows\\Build
     expire_in: 1 day
-  dependencies:
-    - build:windows
   tags:
     - windows
-
 deploy:windows:
   stage: deploy
   script:
-    - .\BuildServer\\Windows\\run_deploy.bat
+    - call .\\BuildServer\\Windows\\deploy.bat
   only:
     - tags
     - develop
@@ -132,7 +128,10 @@ deploy:windows:
   when: on_success
   artifacts:
     paths:
-     - BuildServer/Windows/Build/MDANSE*.exe
+     - BuildServer\\MDANSE*.exe
   tags:
     - windows
-
+clean:windows:
+  stage:clean
+  script:
+    - call .\\BuildServer\\Windows\\clean.bat

BuildServer/.gitignore

+*.deb
+*.dmg
+*.exe
+Unix/Build*
+Windows/Build*
+

BuildServer/Darwin/.gitignore

-Build

BuildServer/Darwin/Scripts/run_build.sh

-#!/bin/bash
-
-# This script is to package the MDANSE package for Mac OS X
-
-#############################
-# CONFIGURATION
-#############################
-
-## Add some colors
-ROUGE="\\033[1;31m"
-BLEU="\\033[1;34m"
-
-##Select the build target
-
-# take the latest version of nmoldyn available on the forge
-echo -e "$BLEU""Getting last MDANSE revision" "$NORMAL"
-
-# Update the __pkginfo__ file with the current commit. The sed -i "" is compulsory other crashes on macos
-COMMIT_ID=$(git rev-parse --short HEAD)
-sed -i "" "s/.*__commit__.*/__commit__ = \"${COMMIT_ID}\"/" MDANSE/__pkginfo__.py
-
-# Get revision number from git (without trailing newline)
-echo -e "$BLEU""Commit id = ${COMMIT_ID}" "$NORMAL"
-
-# Now build last version and install it in our homebrewed python
-echo -e "$BLEU""Building MDANSE" "$NORMAL"
-
-# Clean up temporary build directories
-rm -rf build
-rm -rf dist
-
-# Remove previous install of MDANSE
-rm /Library/Frameworks/Python.framework/Versions/2.7/bin/mdanse*
-rm /Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/MDANSE*.egg-info
-rm -rf /Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/MDANSE
-
-# Build and install MDANSE to the homebrewed python
-/Library/Frameworks/Python.framework/Versions/2.7/bin/python setup.py build >> BuildServer/Darwin/Scripts/build_log.txt 2>&1
-status=$?
-if [ $status -ne 0 ]; then
-	echo -e "$ROUGE" "Failed to build MDANSE" "$NORMAL"
-	exit $status
-fi
-
-/Library/Frameworks/Python.framework/Versions/2.7/bin/python setup.py install >> BuildServer/Darwin/Scripts/build_log.txt 2>&1
-status=$?
-if [ $status -ne 0 ]; then
-    echo -e "$ROUGE" "Failed to install MDANSE" "$NORMAL"
-    exit $status
-fi
-
-exit 0
\ No newline at end of file

BuildServer/Darwin/Scripts/run_tests.sh

-#!/bin/bash
-
-ROUGE="\\033[1;31m"
-
-cd $CI_PROJECT_DIR
-
-export PYTHONPATH=${CI_PROJECT_DIR}/build/lib.macosx-10.6-intel-2.7
-
-# Performs the unit tests
-cd Tests/UnitTests
-/Library/Frameworks/Python.framework/Versions/2.7/bin/nosetests --verbosity=3 -P .
-# Exit now if unable to run tests
-status=$?
-if [ $status -ne 0 ]; then
-	echo -e "$ROUGE""One or several unit tests failed"
-	exit $status
-fi
-cd ../..
-
-# Performs the functional tests
-cd Tests/FunctionalTests/Jobs
-rm -rf Test_*
-/Library/Frameworks/Python.framework/Versions/2.7/bin/python BuildJobTests.py
-/Library/Frameworks/Python.framework/Versions/2.7/bin/nosetests --verbosity=3 --exe Test_*.py
-
-status=$?
-if [ $status -ne 0 ]; then
-	echo -e "$ROUGE""One or several functional tests failed"
-	exit $status
-fi
-
-exit 0
\ No newline at end of file

BuildServer/Darwin/Tools/create-dmg/.gitignore

-.svn

BuildServer/Debian/run_build.sh

-#!/bin/bash
-
-export ARCH=$1
-export DISTRO=$2
-
-#############################
-# CONFIGURATION
-#############################
-
-## Add some colors
-BLEU="\\033[1;34m"
-
-# build ILL version of ScientificPython
-cd /tmp
-rm -rf scientific-python
-git clone https://code.ill.fr/scientific-software/scientific-python.git
-cd scientific-python
-git checkout master
-python setup.py build
-
-declare -x PYTHONPATH=/tmp/scientific-python/build/lib.linux-x86_64-2.7/:${PYTHONPATH}
-
-# build ILL version of ScientificPython
-cd /tmp
-rm -rf mmtk
-git clone https://code.ill.fr/scientific-software/mmtk.git
-cd mmtk
-git checkout master
-python setup.py build
-
-
-declare -x PYTHONPATH=/tmp/mmtk/build/lib.linux-x86_64-2.7/:${PYTHONPATH}
-
-cd
-cd $CI_PROJECT_DIR
-
-# Update the __pkginfo__ file with the current commit 
-COMMIT_ID=$(git rev-parse --short HEAD)
-sed -i "s/.*__commit__.*/__commit__ = \"${COMMIT_ID}\"/" MDANSE/__pkginfo__.py
-
-# Get revision number from git (without trailing newline)
-echo -e "$BLEU""Commit id = ${COMMIT_ID}<--" "$NORMAL"
-
-# Now build last version
-echo "$BLEU""Building MDANSE" "$NORMAL"
-python setup.py build
-
-status=$?
-if [ $status -ne 0 ]; then
-	echo "Failed to build MDANSE"
-	exit $status
-fi
-
-exit 0

BuildServer/Debian/run_tests.sh

-#!/bin/bash
-
-ROUGE="\\033[1;31m"
-
-cd $CI_PROJECT_DIR
-
-export PYTHONPATH=${CI_PROJECT_DIR}/build/lib.linux-x86_64-2.7
-
-# Performs the unit tests
-cd Tests/UnitTests
-nosetests --verbosity=3 -P .
-# Exit now if unable to run tests
-status=$?
-if [ $? -ne 0 ]; then
-	echo -e "$ROUGE""One or several unit tests failed"
-	exit $status
-fi
-cd ../..
-
-# Performs the functional tests
-cd Tests/FunctionalTests/Jobs
-rm -rf Test_*
-python BuildJobTests.py
-nosetests --verbosity=3 --exe Test_*.py
-
-status=$?
-if [ $status -ne 0 ]; then
-	echo -e "$ROUGE""One or several functional tests failed"
-	exit $status
-fi
-
-exit 0
\ No newline at end of file

BuildServer/Darwin/Scripts/build.py → BuildServer/Unix/MacOS_resources/build.py

@@ -7,7 +7,7 @@ if sys.platform.startswith('darwin'):
 
     version = os.environ['VERSION_NAME']
 
-    APP = ['../../../Scripts/mdanse_gui']
+    APP = ['../../Scripts/mdanse_gui']
 
     PLIST = {
         u'CFBundleName': u'MDANSE',
@@ -18,13 +18,13 @@ if sys.platform.startswith('darwin'):
     }
     OPTIONS = {
         'argv_emulation': False,# has to be False otherwise triggers problems zith wxPython which lose some events that are captured by OS
-        'iconfile': u'../../../MDANSE/GUI/Icons/mdanse.icns',
+        'iconfile': u'../../MDANSE/GUI/Icons/mdanse.icns',
         'excludes': 'PyQt4',
 		'matplotlib_backends': '-',
         'optimize': '1',
         'plist': PLIST,
-        'bdist_base': '../Build/build',
-        'dist_dir': '../Build/dist',
+        'bdist_base': './Build/build',
+        'dist_dir': './Build/dist',
         'graph': False,
         'xref': False,
         'packages' : ["MDANSE","MMTK","Scientific"]

BuildServer/Darwin/Scripts/change_dylib_path.sh → BuildServer/Unix/MacOS_resources/change_dylib_path.sh

 #!/bin/bash
 
-files=`ls ./dist/MDANSE.app/Contents/Frameworks/libwx*.dylib`
+files=`ls $MACOS_ROOT_DIR/dist/MDANSE.app/Contents/Frameworks/libwx*.dylib`
 
 libs="osx_cocoau_xrc osx_cocoau_webview osx_cocoau_html osx_cocoau_qa osx_cocoau_adv osx_cocoau_core baseu_xml baseu_net baseu"
 

BuildServer/Unix/build.sh

+#!/bin/bash
+
+#############################
+# PREPARATION
+#############################
+rm -rf build
+
+# Create the temporary directory where ScientificPython, MMTK and MDANSE will be installed
+rm -rf ${MDANSE_TEMPORARY_INSTALLATION_DIR}
+mkdir ${MDANSE_TEMPORARY_INSTALLATION_DIR}
+
+#############################
+# BUILDING DEPENDENCIES
+#############################
+# Build ILL version of ScientificPython
+echo -e "${BLUE}""Building ScientificPython""${NORMAL}"
+cd /tmp
+rm -rf scientific-python
+git clone https://code.ill.fr/scientific-software/scientific-python.git
+cd scientific-python
+git checkout master
+${PYTHONEXE} setup.py build
+status=$?
+if [ $status -ne 0 ]; then
+	echo -e "${RED}" "Failed to build Scientific""${NORMAL}"
+	${MDANSE_SOURCE_DIR}/BuildServer/Unix/clean.sh
+	exit $status
+fi
+${PYTHONEXE} setup.py install --prefix=${MDANSE_TEMPORARY_INSTALLATION_DIR}
+status=$?
+if [ $status -ne 0 ]; then
+	echo -e "${RED}" "Failed to install Scientific""${NORMAL}"
+	${MDANSE_SOURCE_DIR}/BuildServer/Unix/clean.sh
+	exit $status
+fi
+export NETCDF_HEADER_FILE_PATH=${MDANSE_TEMPORARY_INSTALLATION_DIR}/include/python2.7/
+cp ${NETCDF_HEADER_FILE_PATH}/Scientific/netcdf.h ${NETCDF_HEADER_FILE_PATH}
+
+# Build ILL version of MMTK
+echo -e "${BLUE}""Building MMTK""${NORMAL}"
+cd /tmp
+rm -rf mmtk
+git clone https://code.ill.fr/scientific-software/mmtk.git
+cd mmtk
+git checkout master
+${PYTHONEXE} setup.py build
+status=$?
+if [ $status -ne 0 ]; then
+	echo -e "${RED}" "Failed to build MMTK""${NORMAL}"
+	${MDANSE_SOURCE_DIR}/BuildServer/Unix/clean.sh
+	exit $status
+fi
+${PYTHONEXE} setup.py install --prefix=${MDANSE_TEMPORARY_INSTALLATION_DIR}
+status=$?
+if [ $status -ne 0 ]; then
+	echo -e "${RED}" "Failed to install MMTK""${NORMAL}"
+	${MDANSE_SOURCE_DIR}/BuildServer/Unix/clean.sh
+	exit $status
+fi
+
+#############################
+# MDANSE Building
+#############################
+echo -e "${BLUE}""Building MDANSE""${NORMAL}"
+cd $MDANSE_SOURCE_DIR
+
+# Now build last version and install it
+${PYTHONEXE} setup.py build
+status=$?
+if [ $status -ne 0 ]; then
+	echo -e "${RED}" "Failed to build MDANSE""${NORMAL}"
+	${MDANSE_SOURCE_DIR}/BuildServer/Unix/clean.sh
+	exit $status
+fi
+${PYTHONEXE} setup.py install --prefix=${MDANSE_TEMPORARY_INSTALLATION_DIR}
+status=$?
+if [ $status -ne 0 ]; then
+	echo -e "${RED}" "Failed to install MDANSE""${NORMAL}"
+	${MDANSE_SOURCE_DIR}/BuildServer/Unix/clean.sh
+	exit $status
+fi
+
+rm -rf build
\ No newline at end of file

BuildServer/Unix/clean.sh

+#!/bin/bash
+
+#############################
+# PREPARATION
+#############################
+cd ${MDANSE_SOURCE_DIR}
+
+#############################
+# UNITARY TESTS
+#############################
+echo -e "${BLUE}""Cleaning repository""${NORMAL}"
+rm -rf MDANSE_TEMPORARY_INSTALLATION_DIR
+rm -rf build
+rm -rf BuildServer/Build_Debian
+rm -rf BuildServer/Build_macOS
\ No newline at end of file

BuildServer/Unix/definitions_debian.sh

+export ARCH=amd64
+export DISTRO=Xenial
+export PYTHONEXE=/usr/bin/python
+# The sed -i"" is compulsory other crashes on macos
+export SED_I_COMMAND='sed -i'
+export MDANSE_SOURCE_DIR=`pwd`
+export MDANSE_TEMPORARY_INSTALLATION_DIR=${MDANSE_SOURCE_DIR}/BuildServer/Unix/Build
+export PYTHONPATH=${MDANSE_TEMPORARY_INSTALLATION_DIR}/lib/python2.7/site-packages/:${PYTHONPATH}
+export RED="\\033[1;31m"
+export BLUE="\\033[1;34m"
+export NORMAL="\\033[0m"
\ No newline at end of file

BuildServer/Unix/definitions_macos.sh

+export ARCH=amd64
+export DISTRO=macOS
+export PYTHONEXE=/Library/Frameworks/Python.framework/Versions/2.7/bin/python
+# The sed -i"" is compulsory other crashes on macos
+export SED_I_COMMAND='sed -i ""'
+export MDANSE_SOURCE_DIR=`pwd`
+export MDANSE_TEMPORARY_INSTALLATION_DIR=${MDANSE_SOURCE_DIR}/BuildServer/Unix/Build
+export PYTHONPATH=${MDANSE_TEMPORARY_INSTALLATION_DIR}/lib/python2.7/site-packages/:${PYTHONPATH}
+export RED="\\033[1;31m"
+export BLUE="\\033[1;34m"
+export NORMAL="\\033[0m"
\ No newline at end of file

BuildServer/Debian/run_deploy.sh → BuildServer/Unix/deploy_debian.sh

 #!/bin/bash
 
-export ARCH=$1
-export DISTRO=$2
-
 #############################
 # CONFIGURATION
 #############################
+# Debug option for py2app, if needed
+export DISTUTILS_DEBUG=0
 
-## Add some colors
-ROUGE="\\033[1;31m"
-BLEU="\\033[1;34m"
-
-COMMIT_ID=$(git rev-parse --short HEAD)
-VERSION_NAME=`sed -n 's/__version__.*=.*\"\(.*\)\"/\1/p' MDANSE/__pkginfo__.py`
-
-if [[ ${CI_BUILD_REF_NAME} =~ develop ]]
-then
-    if [ -n "${WEEKLY_BUILD}" ]
-    then
-        VERSION_NAME=${VERSION_NAME}-"weekly-"`date +%Y-%m-%d`
-    fi
-    VERSION_NAME=${VERSION_NAME}-${COMMIT_ID}
-fi
-
-##Select the build target
-BUILD_TARGET=debian
-
-cd
-cd $CI_PROJECT_DIR
+#############################
+# PREPARATION
+#############################
+cd ${MDANSE_SOURCE_DIR}
+DEBIAN_ROOT_DIR=${MDANSE_SOURCE_DIR}/BuildServer/Build_Debian
+rm -rf ${DEBIAN_ROOT_DIR}
+mkdir ${DEBIAN_ROOT_DIR}
 
-echo "$BLEU""Build debian tree" "$NORMAL"
+#############################
+# PACKAGING
+#############################
+echo -e "${BLUE}""Build debian tree""${NORMAL}"
 
 # Set automatically the good version number for the Debian control file
-sed -i "s/Version:.*/Version: ${VERSION_NAME}/g" BuildServer/Debian/DEBIAN/control
-
-DEBIAN_ROOT_DIR=BuildServer/Debian/Build
+sed -i "s/Version:.*/Version: ${VERSION_NAME}/g" BuildServer/Unix/Debian_resources/DEBIAN/control
 
 # Copy all the debian files (e.g. control, copyright, md5sum ...) into DEBIAN directory
-mkdir ${DEBIAN_ROOT_DIR}
-cp -r BuildServer/Debian/DEBIAN ${DEBIAN_ROOT_DIR}/
+cp -r BuildServer/Unix/Debian_resources/DEBIAN ${DEBIAN_ROOT_DIR}/
 chmod -R 755 ${DEBIAN_ROOT_DIR}/DEBIAN
 
 # Build the /usr/share/applications directory inside the debian root directory and copy the mdanse desktop file inside
 DEBIAN_APP_DIR=${DEBIAN_ROOT_DIR}/usr/share/applications
 mkdir -p ${DEBIAN_APP_DIR}
-cp BuildServer/Debian/MDANSE.desktop ${DEBIAN_APP_DIR}/
+cp BuildServer/Unix/Debian_resources/MDANSE.desktop ${DEBIAN_APP_DIR}/
 
 # Build the /usr/share/pixmaps directory inside the debian root directory and copy the mdanse icon file inside
 DEBIAN_PIXMAPS_DIR=${DEBIAN_ROOT_DIR}/usr/share/pixmaps
@@ -54,25 +39,29 @@ cp MDANSE/GUI/Icons/mdanse.png ${DEBIAN_PIXMAPS_DIR}/
 # Build the /usr/local/bin directory inside the debian root directory and copy the mdanse scripts inside
 DEBIAN_BIN_DIR=${DEBIAN_ROOT_DIR}/usr/local/bin
 mkdir -p ${DEBIAN_BIN_DIR}
-cp build/scripts-2.7/* ${DEBIAN_BIN_DIR}/
+cp Scripts/* ${DEBIAN_BIN_DIR}/
 dos2unix ${DEBIAN_BIN_DIR}/mdanse_*
 
 # Build the usr/local/lib/python2.7/dist-packages directory inside the debian root directory and copy the MDANSE package inside
 DEBIAN_DIST_DIR=${DEBIAN_ROOT_DIR}/usr/local/lib/python2.7/dist-packages
 mkdir -p ${DEBIAN_DIST_DIR}