Switch staticbuild scripts to Flang Fortran compiler

CMakeLists.txt

@@ -776,6 +776,7 @@ endif()
 
 if(USE_DIST_KVSTORE)
   add_subdirectory("3rdparty/ps-lite")
+  target_compile_options(mxnet PRIVATE "$<$<COMPILE_LANGUAGE:CXX>:-Wno-error=overloaded-virtual>")
   add_definitions(-DMXNET_USE_DIST_KVSTORE)
   list(APPEND mxnet_LINKER_LIBS pslite)
 endif()

cd/mxnet_lib/static/Jenkins_pipeline.groovy

@@ -32,8 +32,8 @@ libmxnet = 'lib/libmxnet.so'
 licenses = 'licenses/*'
 
 // libmxnet dependencies
-mx_native_deps = 'lib/libgfortran.so.4, lib/libquadmath.so.0, lib/libopenblas.so.0'
-mx_deps = 'lib/libgfortran.so.4, lib/libquadmath.so.0, lib/libopenblas.so.0, include/mkldnn/dnnl_version.h, include/mkldnn/dnnl_config.h'
+mx_native_deps = 'lib/libgfortran.so.4, lib/libquadmath.so.0, lib/libopenblas.so.0, lib/libomp.so'
+mx_deps = 'lib/libgfortran.so.4, lib/libquadmath.so.0, lib/libopenblas.so.0, lib/libomp.so, include/mkldnn/dnnl_version.h, include/mkldnn/dnnl_config.h'
 
 // library type
 // either static or dynamic - depending on how it links to its dependencies

ci/build.py

@@ -357,6 +357,9 @@ def main() -> int:
                         default='mxnetci',
                         type=str)
 
+    parser.add_argument("--no-pull", action="store_true",
+                        help="Don't pull from dockerhub registry to initialize cache.")
+
     parser.add_argument("-r", "--docker-build-retries",
                         help="Number of times to retry building the docker image. Default is 1",
                         default=1,
@@ -394,7 +397,7 @@ def main() -> int:
     elif args.platform:
         platform = args.platform
         tag = get_docker_tag(platform=platform, registry=args.docker_registry)
-        if args.docker_registry:
+        if args.docker_registry and not args.no_pull:
             load_docker_cache(tag=tag, docker_registry=args.docker_registry)
         if not args.run_only:
             build_docker(platform=platform, registry=args.docker_registry, num_retries=args.docker_build_retries,

ci/docker/Dockerfile.build.centos7

@@ -53,6 +53,8 @@ RUN yum -y check-update || true && \
         protobuf-devel \
         # CentOS Software Collections https://www.softwarecollections.org
         devtoolset-7 \
+        llvm-toolset-7.0 \
+        llvm-toolset-7.0-libomp-devel \
         rh-python36 \
         rh-maven35 \
         # Libraries
@@ -67,13 +69,10 @@ RUN yum -y check-update || true && \
         gperf \
         libb2-devel \
         libzstd-devel && \
-    yum clean all && \
-    # Centos 7 only provides ninja-build
-    ln -s /usr/bin/ninja-build /usr/bin/ninja
+    yum clean all
 
-# Make GCC7, Python 3.5 and Maven 3.3 Software Collections available by default
-# during build and runtime of this container
-SHELL [ "/usr/bin/scl", "enable", "devtoolset-7", "rh-python36", "rh-maven35" ]
+# Make Clang, Python 3.6 and Maven 3.3 Software Collections available during following build steps
+SHELL [ "/usr/bin/scl", "enable", "llvm-toolset-7.0", "rh-python36", "rh-maven35" ]
 
 # Install minimum required cmake version
 RUN cd /usr/local/src && \
@@ -119,18 +118,26 @@ RUN export SHORT_CUDA_VERSION=${CUDA_VERSION%.*} && \
         yum clean all; \
     fi
 
+# Flang fortran compiler as replacement for gfortran due to the resulting
+# runtime dependency on libgfortran.so which is GPL-licensed and thus
+# https://www.apache.org/legal/resolved.html#category-x for distribution.
+# We need a Fortran compiler to compile the OpenBLAS Lapack implementation.
+# NOTE: The flang toolchain also provides llvm libomp.so and libomp.a. We must
+# use the libomp provided by flang for all parts of the project, to avoid
+# linking multiple omp implementations. Linking 2 different llvm omp's would
+# cause segfaults on importing mxnet. In the next step we'll install compatible
+# omp headers.
+RUN cd /usr/local/ && mkdir flang && cd flang && \
+    wget -nv https://github.com/flang-compiler/flang/releases/download/flang_20190329/flang-20190329-x86-70.tgz && \
+    tar xf flang-20190329-x86-70.tgz && \
+    rm flang-20190329-x86-70.tgz && \
+    # IMPORTANT: Replace libomp.so with the flang version, to avoid segfault upon importing MXNet
+    cp /usr/local/flang/lib/libomp.so /opt/rh/llvm-toolset-7.0/root/usr/lib64/libomp.so
+
 # Python dependencies
-RUN pip3 install --no-cache-dir --upgrade pip && \
-    pip3 install --no-cache-dir pylint cython numpy requests h5py scipy==1.2.3 wheel \
-    pytest==5.3.5 \
-    pytest-env==0.6.2 \
-    pytest-cov==2.8.1 \
-    pytest-xdist==1.31.0 \
-    pytest-timeout==1.3.4 \
-    mock==2.0.0 \
-    onnx==1.5.0 \
-    protobuf==3.5.2 \
-    tabulate==0.7.5
+COPY install/requirements /work/
+RUN python -m pip install --no-cache-dir --upgrade pip && \
+    python -m pip install --no-cache-dir -r /work/requirements
 
 # Fix the en_DK.UTF-8 locale to test locale invariance
 RUN localedef -i en_DK -f UTF-8 en_DK.UTF-8

ci/docker/install/requirements

@@ -35,14 +35,14 @@ Cython==0.29.7
 cpplint==1.3.0
 pylint==2.3.1  # pylint and astroid need to be aligned
 astroid==2.3.3  # pylint and astroid need to be aligned
-pytest==5.3.5
+pytest==5.4.2
 pytest-env==0.6.2
-pytest-cov==2.8.1
-pytest-xdist==1.31.0
+pytest-cov==2.9.0
+pytest-xdist==1.32.0
 pytest-timeout==1.3.4
 flaky==3.6.1
-setuptools
-mock==2.0.0
+setuptools==47.1.1
+wheel==0.34.2
 
 # TVM dependencies
 decorator==4.4.0

ci/docker/runtime_functions.sh

@@ -162,8 +162,8 @@ build_dynamic_libmxnet() {
     # relevant licenses will be placed in the licenses directory
     gather_licenses
 
-    cd /work/build
     source /opt/rh/devtoolset-7/enable
+    cd /work/build
     if [[ ${mxnet_variant} = "cpu" ]]; then
         cmake -DUSE_MKL_IF_AVAILABLE=OFF \
             -DUSE_MKLDNN=ON \
@@ -1929,11 +1929,15 @@ checkout() {
 build_static_libmxnet() {
     set -ex
     pushd .
-    source /opt/rh/devtoolset-7/enable
-    source /opt/rh/rh-python36/enable
     export USE_SYSTEM_CUDA=1
     export CMAKE_STATICBUILD=1
     local mxnet_variant=${1:?"This function requires a python command as the first argument"}
+    if [[ $mxnet_variant == cu* ]]; then
+        source /opt/rh/devtoolset-7/enable
+    else
+        source /opt/rh/llvm-toolset-7.0/enable
+    fi
+    source /opt/rh/rh-python36/enable
     source tools/staticbuild/build.sh ${mxnet_variant}
     popd
 }
@@ -1942,7 +1946,6 @@ build_static_libmxnet() {
 cd_package_pypi() {
     set -ex
     pushd .
-    source /opt/rh/devtoolset-7/enable
     source /opt/rh/rh-python36/enable
     local mxnet_variant=${1:?"This function requires a python command as the first argument"}
     ./cd/python/pypi/pypi_package.sh ${mxnet_variant}
@@ -1996,7 +1999,7 @@ build_static_scala_cpu() {
     scala_prepare
     export MAVEN_PUBLISH_OS_TYPE=linux-x86_64-cpu
     export mxnet_variant=cpu
-    source /opt/rh/devtoolset-7/enable
+    source /opt/rh/llvm-toolset-7.0/enable
     source /opt/rh/rh-maven35/enable
     ./ci/publish/scala/build.sh
     popd
@@ -2006,7 +2009,7 @@ build_static_python_cpu() {
     set -ex
     pushd .
     export mxnet_variant=cpu
-    source /opt/rh/devtoolset-7/enable
+    source /opt/rh/llvm-toolset-7.0/enable
     source /opt/rh/rh-python36/enable
     ./ci/publish/python/build.sh
     popd
@@ -2028,7 +2031,7 @@ build_static_python_cpu_cmake() {
     pushd .
     export mxnet_variant=cpu
     export CMAKE_STATICBUILD=1
-    source /opt/rh/devtoolset-7/enable
+    source /opt/rh/llvm-toolset-7.0/enable
     source /opt/rh/rh-python36/enable
     ./ci/publish/python/build.sh
     popd
@@ -2050,7 +2053,7 @@ publish_scala_build() {
     set -ex
     pushd .
     scala_prepare
-    source /opt/rh/devtoolset-7/enable
+    source /opt/rh/llvm-toolset-7.0/enable
     source /opt/rh/rh-maven35/enable
     export USE_SYSTEM_CUDA=1
     ./ci/publish/scala/build.sh

cmake/ChooseBlas.cmake

@@ -42,6 +42,11 @@ elseif(BLAS STREQUAL "Open" OR BLAS STREQUAL "open")
   find_package(OpenBLAS REQUIRED)
   include_directories(SYSTEM ${OpenBLAS_INCLUDE_DIR})
   list(APPEND mshadow_LINKER_LIBS ${OpenBLAS_LIB})
+  if(CMAKE_BUILD_TYPE STREQUAL "Distribution" AND DEFINED ENV{FC})
+    if($ENV{FC} MATCHES "flang")
+      list(APPEND mshadow_LINKER_LIBS /usr/local/flang/lib/libflang.a /usr/local/flang/lib/libflangrti.a /usr/local/flang/lib/libpgmath.a)
+    endif()
+  endif()
   add_definitions(-DMSHADOW_USE_CBLAS=1)
   add_definitions(-DMSHADOW_USE_MKL=0)
   add_definitions(-DMXNET_USE_BLAS_OPEN=1)

make/staticbuild/linux_cpu.mk

@@ -37,7 +37,7 @@ DEBUG = 0
 USE_SIGNAL_HANDLER = 1
 
 # the additional link flags you want to add
-ADD_LDFLAGS += -L$(DEPS_PATH)/lib -lpng -ltiff -ljpeg -lz -lgfortran -ldl  -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections
+ADD_LDFLAGS += -L$(DEPS_PATH)/lib -lpng -ltiff -ljpeg -lz -lflang -lflangrti -lpgmath -ldl  -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections
 
 # the additional compile flags you want to add
 ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections

make/staticbuild/linux_native.mk

@@ -37,7 +37,7 @@ DEBUG = 0
 USE_SIGNAL_HANDLER = 1
 
 # the additional link flags you want to add
-ADD_LDFLAGS += -L$(DEPS_PATH)/lib -lpng -ltiff -ljpeg -lz -lgfortran -ldl -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections
+ADD_LDFLAGS += -L$(DEPS_PATH)/lib -lpng -ltiff -ljpeg -lz -lflang -lflangrti -lpgmath -ldl -Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections
 
 # the additional compile flags you want to add
 ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections

scala-package/assembly/src/main/assembly/assembly.xml

@@ -55,8 +55,7 @@
       <includes>
         <include>libmxnet.so</include>
         <include>libtvm_runtime.so</include>
-        <include>libgfortran.so.4</include>
-        <include>libquadmath.so.0</include>
+        <include>libomp.so</include>
       </includes>
       <outputDirectory>lib/native</outputDirectory>
     </fileSet>

scala-package/core/src/main/scala/org/apache/mxnet/util/NativeLibraryLoader.scala

@@ -87,8 +87,7 @@ private[mxnet] object NativeLibraryLoader {
     val libFileInJar = libPathInJar + loadLibname
     saveLibraryToTemp("libmxnet.so", "/lib/native/libmxnet.so", true)
     saveLibraryToTemp("libtvm_runtime.so", "/lib/native/libtvm_runtime.so", false)
-    saveLibraryToTemp("libgfortran.so.4", "/lib/native/libgfortran.so.4", false)
-    saveLibraryToTemp("libquadmath.so.0", "/lib/native/libquadmath.so.0", false)
+    saveLibraryToTemp("libomp.so", "/lib/native/libomp.so", false)
     val tempfile: File = saveLibraryToTemp(libname, libFileInJar, true)
 
     loadLibraryFromFile(libname, tempfile)

tools/dependencies/LICENSE.binary.dependencies

@@ -9,10 +9,14 @@
     =======================================================================================
 
     1. cityhash  - For details, see https://github.com/google/cityhash/blob/master/COPYING
-    	Copyright (c) 2011 Google, Inc.
+        Copyright (c) 2011 Google, Inc.
+
     2. protobuf - For details, see https://github.com/protocolbuffers/protobuf/blob/master/LICENSE
         Copyright 2008 Google Inc.  All rights reserved.
 
+    3. llvm openmp - For details, see https://github.com/llvm/llvm-project/blob/release/7.x/openmp/LICENSE.txt
+        Copyright (c) 1997-2016 Intel Corporation  All rights reserved.
+
     =======================================================================================
     Mozilla Public License Version 2.0
     =======================================================================================
@@ -51,6 +55,12 @@
         Copyright (c) 1998-2018 The OpenSSL Project
         Copyright (c) 1995-1998 Eric A. Young, Tim J. Hudson
 
+    2. flang - For details, see https://github.com/flang-compiler/flang/blob/flang_20190329/LICENSE.txt
+        Copyright (c) 2017, NVIDIA CORPORATION.  All rights reserved.
+
+    3. pgmath - For details, see https://github.com/flang-compiler/flang/blob/flang_20190329/runtime/libpgmath/LICENSE.txt
+        Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
+
     =======================================================================================
     Other Licenses
     =======================================================================================

tools/dependencies/libpng.sh

@@ -20,7 +20,7 @@
 # This script builds the static library of libpng that can be used as dependency of mxnet/opencv.
 set -ex
 PNG_VERSION=1.6.35
-if [[ ! -f $DEPS_PATH/lib/libpng.a ]]; then
+if [[ ! -f $DEPS_PATH/lib/libpng.a ]] && [[ ! -f $DEPS_PATH/lib64/libpng.a ]]; then
     # download and build libpng
     >&2 echo "Building libpng..."
     download \

tools/dependencies/libturbojpeg.sh

@@ -25,7 +25,8 @@ if [[ $PLATFORM == 'darwin' ]]; then
     JPEG_NASM_OPTION="-D CMAKE_ASM_NASM_COMPILER=/usr/local/bin/nasm"
 fi
 
-if [[ ! -f $DEPS_PATH/lib/libjpeg.a ]] || [[ ! -f $DEPS_PATH/lib/libturbojpeg.a ]]; then
+if [[ ( ! -f $DEPS_PATH/lib/libjpeg.a ) || ( ! -f $DEPS_PATH/lib/libturbojpeg.a ) ]] && \
+       [[ ( ! -f $DEPS_PATH/lib64/libjpeg.a ) || ( ! -f $DEPS_PATH/lib64/libturbojpeg.a ) ]]; then
     # download and build libjpeg
     >&2 echo "Building libjpeg-turbo..."
     download \

tools/dependencies/openblas.sh

@@ -19,9 +19,8 @@
 
 # This script builds the static library of openblas that can be used as dependency of mxnet.
 set -ex
-OPENBLAS_VERSION=4a4c50a7cef9fa91f14e508722f502d956ad5192
-if [[ ((! -e $DEPS_PATH/lib/libopenblas.a) && -z "$CMAKE_STATICBUILD") ||
-          ((! -e $DEPS_PATH/lib/libopenblas.so) && -v CMAKE_STATICBUILD) ]]; then
+OPENBLAS_VERSION=f7659be4a0e05404b251f17a0eefeabb7d31b284
+if [[ (! -e $DEPS_PATH/lib/libopenblas.a) ]]; then
     # download and build openblas
     >&2 echo "Building openblas..."
 
@@ -33,23 +32,23 @@ if [[ ((! -e $DEPS_PATH/lib/libopenblas.a) && -z "$CMAKE_STATICBUILD") ||
     cd $DEPS_PATH/OpenBLAS-${OPENBLAS_VERSION}
 
     # Adding NO_DYNAMIC=1 flag causes make install to fail
-    CXX="g++ -fPIC" CC="gcc -fPIC" $MAKE DYNAMIC_ARCH=1 DYNAMIC_OLDER=1 USE_OPENMP=1
-
-    if [[ -v CMAKE_STATICBUILD ]]; then
-        # We link and redistribute libopenblas.so for cmake staticbuild
-        # cf https://gitlab.kitware.com/cmake/cmake/issues/16221#note_143330
-        patchelf --set-rpath '$ORIGIN' --force-rpath libopenblas.so
+    if [[ "$FC" == *"flang"* ]]; then
+        LD_LIBRARY_PATH="/usr/local/flang/lib:$LD_LIBRARY_PATH" \
+          LDFLAGS="-L/usr/local/flang/lib" CFLAGS="-I/usr/local/flang/include" \
+          CFLAGS="-fPIC" CXXFLAGS="-fPIC" $MAKE DYNAMIC_ARCH=1 DYNAMIC_OLDER=1 USE_OPENMP=1
+    else
+        CFLAGS="-fPIC" CXXFLAGS="-fPIC" $MAKE DYNAMIC_ARCH=1 DYNAMIC_OLDER=1 USE_OPENMP=1
     fi
 
     $MAKE PREFIX=$DEPS_PATH install
 
-
-    if [[ -z "$CMAKE_STATICBUILD" ]]; then
-        # Manually removing .so to avoid linking against it
-        rm $DEPS_PATH/lib/libopenblas*.so
-    fi
+    # Manually removing .so to avoid linking against it
+    rm $DEPS_PATH/lib/libopenblas*.so
 
     popd
+
+    # Makefile build declares -llapack even though libopenblas provides lapack
+    # symbols. Make sure -llapack doesn't cause any harm.
     if [[ -z "$CMAKE_STATICBUILD" ]]; then
         ln -s libopenblas.a $DEPS_PATH/lib/libcblas.a
         ln -s libopenblas.a $DEPS_PATH/lib/liblapack.a

tools/dependencies/opencv.sh

@@ -46,7 +46,7 @@ if [[ ! -f $DEPS_PATH/lib/libopencv_core.a ]] || [[ ! -f $DEPS_PATH/lib/libopenc
     mkdir -p $DEPS_PATH/opencv-$OPENCV_VERSION/build
     pushd .
     cd $DEPS_PATH/opencv-$OPENCV_VERSION/build
-    CXX="g++ -fPIC" CC="gcc -fPIC" cmake \
+    CFLAGS="-fPIC" CXXFLAGS="-fPIC" cmake \
           -D OPENCV_ENABLE_NONFREE=OFF \
           -D WITH_1394=OFF \
           -D WITH_ARAVIS=OFF \

tools/dependencies/openssl.sh

@@ -20,7 +20,8 @@
 # This script builds the static library of openssl that can be used as dependency of mxnet.
 set -ex
 OPENSSL_VERSION=1.1.1b
-if [[ ! -f $DEPS_PATH/lib/libssl.a ]] || [[ ! -f $DEPS_PATH/lib/libcrypto.a ]]; then
+if [[ ( ! -f $DEPS_PATH/lib/libssl.a ) || ( ! -f $DEPS_PATH/lib/libcrypto.a ) ]] && \
+       [[ ( ! -f $DEPS_PATH/lib64/libssl.a ) || ( ! -f $DEPS_PATH/lib64/libcrypto.a ) ]]; then
     # download and build openssl
     >&2 echo "Building openssl..."
     OPENSSL_VERSION=$(echo $OPENSSL_VERSION | sed 's/\./_/g')

tools/dependencies/protobuf.sh

@@ -19,7 +19,7 @@
 
 # This script builds the static library of protobuf along with protoc, that can be used as dependency of mxnet.
 set -ex
-PROTOBUF_VERSION=3.5.1
+PROTOBUF_VERSION=3.12.3
 if [[ $PLATFORM == 'darwin' ]]; then
     DY_EXT="dylib"
 else
@@ -32,13 +32,14 @@ if [[ ! -e $LIBPROTOBUF ]] || [[ ! -e $LIBPROTOC ]]; then
     # Download and build protobuf
     >&2 echo "Building protobuf..."
     download \
-        https://github.com/google/protobuf/archive/v${PROTOBUF_VERSION}.zip \
+        https://github.com/protocolbuffers/protobuf/archive/v${PROTOBUF_VERSION}.zip \
         ${DEPS_PATH}/protobuf.zip
     unzip -q $DEPS_PATH/protobuf.zip -d $DEPS_PATH
     pushd .
     cd $DEPS_PATH/protobuf-$PROTOBUF_VERSION
     ./autogen.sh
-    ./configure -prefix=$DEPS_PATH
+    # Need to find /opt/rh/devtoolset-8/root/usr/libexec/gcc/x86_64-redhat-linux/8/cc1plus
+    PATH=/opt/rh/devtoolset-8/root/usr/libexec/gcc/x86_64-redhat-linux/8/:$PATH ./configure -prefix=$DEPS_PATH
     $MAKE
     $MAKE install
     popd