Skip to content

GitLab

Unverified Commit 0e15b47b authored by Simon Ward's avatar Simon Ward
Browse files

More reworking

parent b6a93dc1
Hide whitespace changes
Inline Side-by-side
.github/workflows/build_many_linux.yml
View file @ 0e15b47b
......@@ -6,7 +6,7 @@ on:
branches: [ simon ]
jobs:
deploy:
create-Linux:
runs-on: ubuntu-latest
steps:
......@@ -24,12 +24,45 @@ jobs:
- name: Build manylinux wheels
uses: RalfG/python-wheels-manylinux-build@v0.2.2-manylinux2010_x86_64
with:
python-versions: 'cp36-cp36m cp37-cp37m cp38-cp38'
python-versions: 'cp36-cp36m cp37-cp37m cp38-cp38 cp310-cp310'
build-requirements: 'cmake'
system-packages: ''
package-path: ''
pip-wheel-args: '-w ./dist'
- uses: actions/upload-artifact@v2
with:
name: CrysFML
name: CrysFML - Linux
path: ${{ github.workspace }}/dist/*
create-OSX:
strategy:
max-parallel: 4
matrix:
python-version: [3.6, 3.7, 3.8, 3.9]
os: [macos-10.15, macos-11.0]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v2
- name: Setup python
uses: actions/setup-python@v1
with:
python-version: ${{matrix.python-version}}
- name: Set Compiler (OSX)
run: |
echo "FC=gfortran-9" >> $GITHUB_ENV
echo "CC=gcc-9" >> $GITHUB_ENV
- name: Install cmake
run: pip3 install cmake
- name: Build CFML
run: pip wheel ./ --no-deps -w ./dist/
- uses: actions/upload-artifact@v2
with:
name: CrysFML - OSX
path: ${{ github.workspace }}/dist/*
\ No newline at end of file
Python_API/Src/CMakeLists.txt
View file @ 0e15b47b
......@@ -40,40 +40,24 @@ if (NOT PYTHON_INTERPRETER_PATH)
else()
message(STATUS "using Python interpreter provided with PYTHON_INTERPRETER_PATH variable: " ${PYTHON_INTERPRETER_PATH})
endif()
#if (NOT PYTHON_LIBRARY_PATH)
# set(Python_ADDITIONAL_VERSIONS ${COMPATIBLE_VERSIONS})
# find_package(PythonLibs REQUIRED)
# set(PYTHON_LIBRARY_PATH ${PYTHON_LIBRARY})
# message(STATUS ${PYTHON_LIBRARY_PATH})
#else()
# message(STATUS "Linking Python extension with provided PYTHON_LIBRARY_PATH variable: " ${PYTHON_LIBRARY_PATH})
#
# if (NOT PYTHON_LIBRARY_PATH)
# set(Python_ADDITIONAL_VERSIONS ${COMPATIBLE_VERSIONS})
# find_package(PythonLibs REQUIRED)
# set(PYTHON_LIBRARY_PATH ${PYTHON_LIBRARY})
# message(STATUS ${PYTHON_LIBRARY_PATH})
# else()
# message(STATUS "Linking Python extension with provided PYTHON_LIBRARY_PATH variable: " ${PYTHON_LIBRARY_PATH})
# endif()
# if (UNIX)
# add_library(Python3_LIB SHARED IMPORTED)
# set_property(TARGET Python3_LIB PROPERTY IMPORTED_LOCATION ${PYTHON_LIBRARY_PATH})
# add_library(Python3_LIB SHARED IMPORTED)
# set_property(TARGET Python3_LIB PROPERTY IMPORTED_LOCATION ${PYTHON_LIBRARY_PATH})
# elseif(WIN32)
# add_library(Python3_LIB STATIC IMPORTED)
# set_property(TARGET Python3_LIB PROPERTY IMPORTED_LOCATION ${PYTHON_LIBRARY_PATH})
# add_library(Python3_LIB STATIC IMPORTED)
# set_property(TARGET Python3_LIB PROPERTY IMPORTED_LOCATION ${PYTHON_LIBRARY_PATH})
# endif()
# Set include and linking
include_directories(${CRYSFML_COMMON_MODULE_DIRECTORY})
target_link_libraries(${LIBRARY_NAME} crysfml_common)
# target_link_libraries(${LIBRARY_NAME} Python3_LIB)
# Set library extension
set_target_properties(${LIBRARY_NAME} PROPERTIES PREFIX "" OUTPUT_NAME ${LIBRARY_NAME})
set(LIBRARY_FILENAME "crysfml_api.so")
if(APPLE)
set_target_properties(${LIBRARY_NAME} PROPERTIES SUFFIX ".so")
set(LIBRARY_FILENAME "crysfml_api.so")
elseif(WIN32)
set_target_properties(${LIBRARY_NAME} PROPERTIES SUFFIX ".pyd")
set(LIBRARY_FILENAME "crysfml_api.pyd")
endif()
#################################
# Documentation
#################################
......@@ -117,12 +101,12 @@ add_custom_target(Python_Files ALL
${CMAKE_COMMAND} -E copy_directory
${CMAKE_CURRENT_SOURCE_DIR}/../Tests
${CMAKE_CURRENT_BINARY_DIR}/../Tests
COMMAND
${CMAKE_COMMAND} -E copy_directory
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_BINARY_DIR}/../Tests/CFML_api
COMMAND
${CMAKE_COMMAND} -E copy
${CMAKE_CURRENT_BINARY_DIR}/${LIBRARY_FILENAME}
......@@ -130,6 +114,27 @@ add_custom_target(Python_Files ALL
)
add_dependencies(Python_Files ${LIBRARY_NAME})
# Set -Wl,-undefined,dynamic_lookup to OTHER_LDFLAGS
# Set include and linking
include_directories(${CRYSFML_COMMON_MODULE_DIRECTORY})
target_link_libraries(${LIBRARY_NAME} crysfml_common)
# Fix a problem on Mac OS X when building shared libraries
if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -undefined dynamic_lookup")
endif()
# target_link_libraries(${LIBRARY_NAME} Python3_LIB)
# Set library extension
set_target_properties(${LIBRARY_NAME} PROPERTIES PREFIX "" OUTPUT_NAME ${LIBRARY_NAME})
set(LIBRARY_FILENAME "crysfml_api.so")
if(APPLE)
set_target_properties(${LIBRARY_NAME} PROPERTIES SUFFIX ".so")
set(LIBRARY_FILENAME "crysfml_api.so")
elseif(WIN32)
set_target_properties(${LIBRARY_NAME} PROPERTIES SUFFIX ".pyd")
set(LIBRARY_FILENAME "crysfml_api.pyd")
endif()
#################################
# Install section
......@@ -145,9 +150,9 @@ endif()
# Installing example files
install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../Examples" DESTINATION ${PYTHON_API_PREFIX} FILES_MATCHING PATTERN "*.py")
install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../Examples/Data" DESTINATION ${PYTHON_API_PREFIX}/Examples FILES_MATCHING PATTERN "*.cfl" PATTERN "*.cif")
# Installing tests files
#
# # Installing tests files
install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../Tests" DESTINATION ${PYTHON_API_PREFIX} FILES_MATCHING PATTERN "*.py")
# Installing documentation files
install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html" DESTINATION ${PYTHON_API_PREFIX}/Doc OPTIONAL)
#
# # Installing documentation files
install(DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/doxygen/html" DESTINATION ${PYTHON_API_PREFIX}/Doc OPTIONAL)
\ No newline at end of file
buildscript.sh 0 → 100755
View file @ 0e15b47b
#!/bin/bash
set -e -u -x
function repair_wheel {
wheel="$1"
if ! auditwheel show "$wheel"; then
echo "Skipping non-platform wheel $wheel"
else
auditwheel repair "$wheel" --plat "$PLAT" -w /io/dist/
fi
}
# Install a system package required by our library
#yum install -y atlas-devel
# yum install -y libaec-dev libz-dev libsz2
# Compile wheels
for PYBIN in /opt/python/*/bin; do
# "${PYBIN}/pip" install -r /io/dev-requirements.txt
"${PYBIN}/pip" wheel /io/ --no-deps -w /io/dist/
rm -rf /io/build/
done
# Bundle external shared libraries into the wheels
for whl in /io/dist/*.whl; do
repair_wheel "$whl"
done
# Install packages and test
for PYBIN in /opt/python/*/bin/; do
"${PYBIN}/pip" install CFML --no-index -f /io/dist
# (cd "$HOME"; "${PYBIN}/nosetests" pymanylinuxdemo)
done
\ No newline at end of file
setup.py
View file @ 0e15b47b
# -*- coding: utf-8 -*-
import os
import sys
from distutils.command.install_data import install_data
from setuptools import find_packages, setup, Extension
from setuptools.command.build_ext import build_ext
from setuptools.command.install_lib import install_lib
from setuptools.command.install_scripts import install_scripts
from subprocess import CalledProcessError, check_output, check_call
import struct
import pkgutil
import re
from setuptools import setup, Extension, find_packages
from setuptools.command.build_ext import build_ext
from distutils.version import LooseVersion
from distutils.sysconfig import get_python_inc
import distutils.sysconfig as sysconfig
# Convert distutils Windows platform specifiers to CMake -A arguments
PLAT_TO_CMAKE = {
"win32": "Win32",
"win-amd64": "x64",
"win-arm32": "ARM",
"win-arm64": "ARM64",
}
import pathlib
import os
import sys
import shutil
BITS = struct.calcsize("P") * 8
PACKAGE_NAME = "crysfml_api"
SOURCE_DIR = '.'
COMPILER = 'gfortran'
# We can use cmake provided from pip which (normally) gets installed at /bin
# Except that in the manylinux builds it's placed at /opt/python/[version]/bin/
......@@ -31,81 +29,254 @@ else:
def get_cmake():
return CMAKE_BIN
# A CMakeExtension needs a sourcedir instead of a file list.
# The name must be the _single_ output extension from the CMake build.
# If you need multiple extensions, see scikit-build.
class CMakeExtension(Extension):
def __init__(self, name, sourcedir=""):
Extension.__init__(self, name, sources=[])
self.sourcedir = os.path.abspath(sourcedir)
"""
An extension to run the cmake build
This simply overrides the base extension class so that setuptools
doesn't try to build your sources for you
"""
def __init__(self, name, sources=[]):
class CMakeBuild(build_ext):
super().__init__(name = name, sources = sources)
class InstallCMakeLibsData(install_data):
"""
Just a wrapper to get the install data into the egg-info
Listing the installed files in the egg-info guarantees that
all of the package files will be uninstalled when the user
uninstalls your package through pip
"""
def run(self):
try:
out = check_output([get_cmake(), '--version'])
except OSError:
raise RuntimeError("CMake must be installed to build" +
" the following extensions: " +
", ".join(e.name for e in self.extensions))
"""
Outfiles are the libraries that were built using cmake
"""
# There seems to be no other way to do this; I tried listing the
# libraries during the execution of the InstallCMakeLibs.run() but
# setuptools never tracked them, seems like setuptools wants to
# track the libraries through package data more than anything...
# help would be appriciated
self.outfiles = self.distribution.data_files
class InstallCMakeLibs(install_lib):
"""
Get the libraries from the parent distribution, use those as the outfiles
Skip building anything; everything is already built, forward libraries to
the installation step
"""
def run(self):
"""
Copy libraries from the bin directory and place them as appropriate
"""
self.announce("Moving library files", level=3)
# We have already built the libraries in the previous build_ext step
self.skip_build = True
# bin_dir = self.distribution.bin_dir
# Depending on the files that are generated from your cmake
# build chain, you may need to change the below code, such that
# your files are moved to the appropriate location when the installation
# is run
bin_dir = os.path.abspath(os.path.join(self.distribution.bin_dir, '..'))
libs = [os.path.join(bin_dir, _dir) for _dir in
os.listdir(bin_dir) if
os.path.isdir(os.path.join(bin_dir, _dir))]
for lib in libs:
shutil.move(lib, os.path.join(self.build_dir,
os.path.basename(lib)))
# Move the lib to the correct location.
bin_dir = self.build_dir
pyd_path = [os.path.join(bin_dir, _pyd) for _pyd in
os.listdir(bin_dir) if
os.path.isfile(os.path.join(bin_dir, _pyd)) and
os.path.splitext(_pyd)[0].startswith(PACKAGE_NAME) and
os.path.splitext(_pyd)[1] in [".pyd", ".so"]][0]
shutil.move(pyd_path, os.path.join(os.path.split(pyd_path)[0], 'CFML_api', os.path.split(pyd_path)[1]))
# Mark the libs for installation, adding them to
# distribution.data_files seems to ensure that setuptools' record
# writer appends them to installed-files.txt in the package's egg-info
#
# Also tried adding the libraries to the distribution.libraries list,
# but that never seemed to add them to the installed-files.txt in the
# egg-info, and the online recommendation seems to be adding libraries
# into eager_resources in the call to setup(), which I think puts them
# in data_files anyways.
#
# What is the best way?
# These are the additional installation files that should be
# included in the package, but are resultant of the cmake build
# step; depending on the files that are generated from your cmake
# build chain, you may need to modify the below code
self.distribution.data_files = [os.path.join(self.install_dir,
os.path.basename(lib))
for lib in libs]
rex = r'version\s*([\d.]+)'
cmake_version = LooseVersion(re.search(rex, out.decode()).group(1))
if cmake_version < '3.13.0':
raise RuntimeError("CMake >= 3.13.0 is required")
# Must be forced to run after adding the libs to data_files
for ext in self.extensions:
self.build_extension(ext)
self.distribution.run_command("install_data")
def build_extension(self, ext):
extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name)))
super().run()
# required for auto-detection of auxiliary "native" libs
if not extdir.endswith(os.path.sep):
extdir += os.path.sep
class InstallCMakeScripts(install_scripts):
"""
Install the scripts in the build dir
"""
def run(self):
"""
Copy the required directory to the build directory and super().run()
"""
self.announce("Moving scripts files", level=3)
# Scripts were already built in a previous step
self.skip_build = True
bin_dir = self.distribution.bin_dir
scripts_dirs = []
# scripts_dirs = [os.path.join(bin_dir, _dir) for _dir in
# os.listdir(bin_dir) if
# os.path.isdir(os.path.join(bin_dir, _dir))]
#
# for scripts_dir in scripts_dirs:
#
# shutil.move(scripts_dir,
# os.path.join(self.build_dir,
# os.path.basename(scripts_dir)))
# Mark the scripts for installation, adding them to
# distribution.scripts seems to ensure that the setuptools' record
# writer appends them to installed-files.txt in the package's egg-info
self.distribution.scripts = scripts_dirs
super().run()
class BuildCMakeExt(build_ext):
"""
Builds using cmake instead of the python setuptools implicit build
"""
def run(self):
"""
Perform build_cmake before doing the 'normal' stuff
"""
for extension in self.extensions:
self.build_cmake(extension)
super().run()
def build_cmake(self, extension: Extension):
"""
The steps required to build the extension
"""
cfg = "Debug" if self.debug else "Release"
self.announce("Preparing the build environment", level=3)
build_dir = pathlib.Path(self.build_temp)
extension_path = pathlib.Path(self.get_ext_fullpath(extension.name))
os.makedirs(build_dir, exist_ok=True)
os.makedirs(extension_path.parent.absolute(), exist_ok=True)
# Now that the necessary directories are created, build
self.announce("Configuring cmake project", level=3)
# Change your cmake arguments below as necessary
# Below is just an example set of arguments for building Blender as a Python module
cmake_args = [
'-H' + SOURCE_DIR,
'-B' + self.build_temp,
"-DPYTHON_EXECUTABLE:FILEPATH={}".format(sys.executable),
"-DPython_ROOT_DIR={}/".format(sys.base_prefix),
"-DPYTHON_INTERPRETER_PATH={}".format(sys.executable),
"-DPYTHON_INCLUDE_DIR={}".format(get_python_inc()),
"-DPYTHON_LIBRARY ={}".format(sysconfig.get_config_var('LIBDIR')),
"-DARCH32=OFF",
"-DCMAKE_Fortran_COMPILER=gfortran",
"-DCMAKE_Fortran_COMPILER={}".format(COMPILER),
"-DPYTHON_API=ON",
"-DUSE_HDF=OFF",
"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={}".format(extdir),
"-DEXAMPLE_VERSION_INFO={}".format(self.distribution.get_version()),
"-DCMAKE_BUILD_TYPE={}".format(cfg), # not used on MSVC, but no harm
"-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={}".format(os.path.join(build_dir), 'Release'),
]
build_args = []
if not os.path.exists(self.build_temp):
os.makedirs(self.build_temp)
check_call(
[get_cmake(), ext.sourcedir] + cmake_args, cwd=self.build_temp
[get_cmake()] + cmake_args
)
self.announce("Building binaries", level=3)
check_call(
[get_cmake(), "--build", "."] + build_args, cwd=self.build_temp
[get_cmake(), "--build", ".", '--target', 'install'] + build_args, cwd=self.build_temp
)
# Build finished, now copy the files into the copy directory
# The copy directory is the parent directory of the extension (.pyd)
self.announce("Moving built python module", level=3)
bin_dir = os.path.join(os.getcwd(), COMPILER, 'Python_API', 'CFML_api')
self.distribution.bin_dir = bin_dir
pyd_path = [os.path.join(bin_dir, _pyd) for _pyd in
os.listdir(bin_dir) if
os.path.isfile(os.path.join(bin_dir, _pyd)) and
os.path.splitext(_pyd)[0].startswith(PACKAGE_NAME) and
os.path.splitext(_pyd)[1] in [".pyd", ".so"]][0]
shutil.move(pyd_path, extension_path)
# After build_ext is run, the following commands will run:
#
# install_lib
# install_scripts
#
# These commands are subclassed above to avoid pitfalls that
# setuptools tries to impose when installing these, as it usually
# wants to build those libs and scripts as well or move them to a
# different place. See comments above for additional information
# The information here can also be placed in setup.cfg - better separation of
# logic and declaration, and simpler if you include description/version in a file.
setup(
name="CFML",
version="0.0.1",
author="Simon Ward",
author_email="simon.ward@ess.eu",
description="ManyLinux test of CrysFML",
long_description="",
ext_modules=[CMakeExtension("cmake_example")],
packages=find_packages(),
cmdclass={"build_ext": CMakeBuild},
zip_safe=False,
)
setup(name="CFML",
version="0.0.1",
author="Simon Ward",
author_email="simon.ward@ess.eu",
description="ManyLinux test of CrysFML",
ext_modules=[CMakeExtension(name="crysfml_api")],
long_description=open("./README", 'r').read(),
long_description_content_type="text/markdown",
keywords="crystolography, physics, neutron, diffraction",
classifiers=["Intended Audience :: Developers",
"License :: OSI Approved :: "
"GNU Lesser General Public License v3 (LGPLv3)",
"Natural Language :: English",
"Programming Language :: C",
"Programming Language :: C++",
"Programming Language :: Python"],
license='GPL-3.0',
cmdclass={
'build_ext': BuildCMakeExt,
'install_data': InstallCMakeLibsData,
'install_lib': InstallCMakeLibs,
'install_scripts': InstallCMakeScripts
}
)
setup_old.py 0 → 100644
View file @ 0e15b47b
# -*- coding: utf-8 -*-
import os
import sys
from subprocess import CalledProcessError, check_output, check_call
import pkgutil
import re
from setuptools import setup, Extension, find_packages
from setuptools.command.build_ext import build_ext
from distutils.version import LooseVersion
from distutils.sysconfig import get_python_inc
import distutils.sysconfig as sysconfig
# Convert distutils Windows platform specifiers to CMake -A arguments
PLAT_TO_CMAKE = {
"win32": "Win32",
"win-amd64": "x64",
"win-arm32": "ARM",
"win-arm64": "ARM64",
}
# We can use cmake provided from pip which (normally) gets installed at /bin
# Except that in the manylinux builds it's placed at /opt/python/[version]/bin/
# (as a symlink at least) which is *not* on the path.
# If cmake is a known module, import it and use it tell us its binary directory
if pkgutil.find_loader('cmake') is not None:
import cmake
CMAKE_BIN = cmake.CMAKE_BIN_DIR + os.path.sep + 'cmake'
else:
CMAKE_BIN = 'cmake'
def get_cmake():
return CMAKE_BIN
# A CMakeExtension needs a sourcedir instead of a file list.
# The name must be the _single_ output extension from the CMake build.
# If you need multiple extensions, see scikit-build.
class CMakeExtension(Extension):
def __init__(self, name, sourcedir=""):
Extension.__init__(self, name, sources=[])
self.sourcedir = os.path.abspath(sourcedir)
# A CMakeExtension needs a sourcedir instead of a file list.
# The name must be the _single_ output extension from the CMake build.
# If you need multiple extensions, see scikit-build.
class PythonExtension(Extension):
def __init__(self, name, sourcedir=""):
Extension.__init__(self, name, sources=[])
self.sourcedir = os.path.join('Python_API', 'Src')
class CMakeBuild(build_ext):
def run(self):
try:
out = check_output([get_cmake(), '--version'])
except OSError:
raise RuntimeError("CMake must be installed to build" +
" the following extensions: " +
", ".join(e.name for e in self.extensions))
rex = r'version\s*([\d.]+)'
cmake_version = LooseVersion(re.search(rex, out.decode()).group(1))
if cmake_version < '3.13.0':
raise RuntimeError("CMake >= 3.13.0 is required")
for ext in self.extensions:
self.build_extension(ext)
def build_extension(self, ext):
extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name)))
# required for auto-detection of auxiliary "native" libs
if not extdir.endswith(os.path.sep):
extdir += os.path.sep
cfg = "Debug" if self.debug else "Release"
cmake_args = [
"-DPYTHON_EXECUTABLE:FILEPATH={}".format(sys.executable),
"-DARCH32=OFF",
"-DCMAKE_Fortran_COMPILER=gfortran",