BLD: Scipy meson build with Nix

.github/workflows/linux.yml

@@ -3,92 +3,55 @@ name: Linux Tests
 on:
   push:
     branches:
-      - master
-      - maintenance/**
+      - meson
   pull_request:
     branches:
-      - master
-      - maintenance/**
+      - meson
 
 jobs:
-  Python-37-dbg:
-    name: Python 3.7-dbg
-    if: "github.repository == 'scipy/scipy' && !contains(github.event.head_commit.message, '[ci skip]') && !contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, '[skip github]')"
-    runs-on: ubuntu-18.04
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: recursive
-      - name: Configuring Test Environment
-        run: |
-          sudo apt-get update
-          sudo apt install python3.7-dbg python3.7-dev libatlas-base-dev liblapack-dev gfortran libgmp-dev libmpfr-dev libsuitesparse-dev ccache swig libmpc-dev
-          free -m
-          python3.7-dbg --version # just to check
-          export NPY_NUM_BUILD_JOBS=2
-          uname -a
-          df -h
-          ulimit -a
-          set -e -o pipefail
-          mkdir builds
-          cd builds
-      - name: Installing packages
-        run: |      
-          python3.7-dbg -c 'import sys; print("Python debug build:", hasattr(sys, "gettotalrefcount"))'
-          python3.7-dbg -m pip install --upgrade pip setuptools wheel
-          python3.7-dbg -m pip install --upgrade numpy cython pytest pytest-xdist pybind11
-          python3.7-dbg -m pip install --upgrade mpmath gmpy2 pythran
-          python3.7-dbg -m pip uninstall -y nose
-          cd ..
-      - name: Building SciPy
-        run: python3.7-dbg -u runtests.py -g -j2 --build-only
-      - name: Testing SciPy
-        if: ${{ github.event_name == 'pull_request' }}
-        run: |
-          python3.7-dbg -u runtests.py -g -j2 -m fast -- -rfEX --durations=10 2>&1 | tee runtests.log
-          python3.7-dbg tools/validate_runtests_log.py fast < runtests.log
-      - name: Dynamic symbol hiding check on Linux
-        if: ${{ github.event_name == 'pull_request' }}
-        run: ./tools/check_pyext_symbol_hiding.sh build
-
-  test_nightly:
-    name: Nightly CPython
-    if: "github.repository == 'scipy/scipy' && !contains(github.event.head_commit.message, '[ci skip]') && !contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, '[skip github]') && !contains(github.ref, 'maintenance/') && !contains(github.base_ref, 'maintenance/')"
+  test_meson:
+    name: Meson build
+    if: "github.repository == 'rgommers/scipy' && !contains(github.event.head_commit.message, '[ci skip]') && !contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, '[skip github]') && !contains(github.ref, 'maintenance/') && !contains(github.base_ref, 'maintenance/')"
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.10]
+        python-version: [3.9]
 
     steps:
     - uses: actions/checkout@v2
       with:
         submodules: recursive
 
-    - name: Install Python 3.10
-      if: matrix.python-version == '3.10'
+    - name: Setup Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install Ubuntu dependencies
       run: |
-        sudo add-apt-repository ppa:deadsnakes/ppa
-        sudo apt-get update
-        sudo apt install -y --no-install-recommends python3.10-dev python3.10-distutils python3.10-venv
-        # GitHub doesn't provide a pip interface with py3.10 yet. Therefore, install it manually :
-        curl -O https://bootstrap.pypa.io/get-pip.py && python3.10 get-pip.py && rm get-pip.py
-        python3.10 -m pip install --upgrade pip setuptools
+        # NOTE: not the same OpenBLAS version as in upstream CI (I'm being lazy here)
+        sudo apt-get install libopenblas-dev libatlas-base-dev liblapack-dev gfortran libgmp-dev libmpfr-dev libsuitesparse-dev ccache libmpc-dev
 
-    - name: Install other build dependencies
+    - name: Install Python packages
       run: |
-        sudo apt-get install libatlas-base-dev liblapack-dev gfortran libgmp-dev libmpfr-dev libsuitesparse-dev ccache libmpc-dev
+        python -m pip install numpy setuptools wheel cython pytest pytest-xdist pybind11 pytest-xdist mpmath gmpy2 pythran ninja
+        python -m pip install --user git+https://github.com/rgommers/meson.git@fix-cython-mixed-sources
 
-    - name: Install packages
+    - name: Build SciPy
       run: |
-        pip install --user git+https://github.com/numpy/numpy.git
-        python3.10 -m pip install --user setuptools wheel cython pytest pybind11 pytest-xdist
-        pip install --user git+https://github.com/serge-sans-paille/pythran.git
-        python3.10 -m pip install -r mypy_requirements.txt
+        meson setup build --prefix=$PWD/installdir
+        ninja -C build
 
-    - name: Mypy
+    - name: Install SciPy
       run: |
-        python3.10 -u runtests.py --mypy
+        meson install -C build
 
     - name: Test SciPy
       run: |
-        python3.10 -u runtests.py -m fast
+        export PYTHONPATH=$PWD/installdir/lib/python3.9/site-packages/
+        # FIXME: run all tests, not just one simple module - create a dev.py
+        #        interface. Also ensure that fails CI if tests fail (see runtests.py)
+        # Avoid being in a directory which has a scipy/ dir
+        pushd installdir
+        python -c "import scipy; import sys; sys.exit(int(not scipy.test(parallel=2)))"
+        popd

.gitignore

@@ -244,6 +244,7 @@ scipy/sparse/sparsetools/csr_impl.h
 scipy/sparse/sparsetools/other_impl.h
 scipy/sparse/sparsetools/sparsetools_impl.h
 scipy/spatial/ckdtree.cxx
+scipy/spatial/ckdtree.cpp
 scipy/spatial/ckdtree.h
 scipy/spatial/_hausdorff.c
 scipy/spatial/qhull.c
@@ -289,3 +290,13 @@ scipy/optimize/cython_optimize/_zeros.pyx
 scipy/optimize/lbfgsb/_lbfgsbmodule.c
 scipy/optimize/_highs/cython/src/_highs_wrapper.cxx
 scipy/optimize/_highs/cython/src/_highs_constants.cxx
+scipy/optimize/_highs/cython/src/_highs_wrapper.cpp
+scipy/optimize/_highs/cython/src/_highs_constants.cpp
+scipy/optimize/_highs/cython/src/_highs_wrapper.c
+scipy/optimize/_highs/cython/src/_highs_constants.c
+scipy/stats/_qmc_cy.cxx
+scipy/stats/_qmc_cy.cpp
+scipy/stats/_qmc_cy.c
+scipy/stats/biasedurn.cxx
+scipy/stats/biasedurn.cpp
+scipy/stats/biasedurn.c

MESON_BUILD.md

@@ -0,0 +1,109 @@
+# How to build SciPy with Meson
+
+_note: these instructions are for Linux, if you are on another OS things likely
+don't work yet! macOS support should be easy, please comment if we need to
+prioritize adding that._
+
+## quickstart from scratch
+
+Clone the repo and check out the right branch:
+
+```bash
+git clone [email protected]:rgommers/scipy.git
+git submodule update --init
+git checkout meson
+```
+
+Create a conda development environment, build SciPy with Meson and run the test
+suite:
+
+```
+conda env create -f environment.yml
+conda activate scipy-dev
+python -m pip install git+https://github.com/rgommers/meson.git@scipy
+./mesondev.sh build --prefix=$PWD/install
+```
+
+## Full details and explanation
+
+Now to build SciPy, we need the `meson` branch from `@rgommers`'s fork (this is
+the integration branch until Meson support is merged into SciPy master):
+```bash
+git clone [email protected]:rgommers/scipy.git
+git submodule update --init
+git checkout meson
+```
+
+We will use conda here, because it's the easiest way to get a fully
+reproducible environment. If you do not have a conda environment yet, the
+recommended installer is
+[Mambaforge](https://github.com/conda-forge/miniforge#mambaforge) (`mamba` is
+basically a much faster `conda`).
+
+To create a development environment:
+```bash
+conda env create -f environment.yml  # `mamba` works too for this command
+conda activate scipy-dev
+```
+
+The one dependency that we now miss is `meson`. Support for Cython in Meson is
+very new, and we expect to also need some bug fixes and new features in the
+Meson master branch as well as fixes we may not even have submitted yet or are
+pending review. So install Meson from the `scipy` branch of `@rgommers`'s fork:
+```bash
+python -m pip install git+https://github.com/rgommers/meson.git@scipy
+```
+
+Meson uses a configure and a build stage. To configure it for putting the build
+artifacts in `build/` and a local install under `installdir/` and then build:
+```bash
+meson setup build --prefix=$PWD/installdir
+ninja -C build
+```
+In the command above, `-C` is followed by the name of the build directory. You
+can have multiple builds at the same time. Meson is fully out-of-place, so
+those builds will not interfere with each other. You can for example have a GCC
+build, a Clang build and a debug build in different directories.
+
+To then install SciPy into the prefix (`installdir/` here):
+```bash
+meson install -C build
+```
+It will then install to `installdir/lib/python3.9/site-packages/scipy`, which
+is not on your Python path, so to add it do:
+```bash
+export PYTHONPATH=$PWD/installdir/lib/python3.9/site-packages/
+```
+
+Now we should be able to import `scipy` and run the tests. Remembering that we
+need to move out of the root of the repo to ensure we pick up the package and
+not the local `scipy/` source directory:
+```bash
+cd doc
+python -c "from scipy import constants as s; s.test()"
+```
+The above runs the tests for a single module, `constants`. Other ways of
+running the tests should also work, for example:
+```bash
+pytest scipy
+```
+
+Current status (24 July) is that the full test suite passes on Linux with
+OpenBLAS. And there is CI on this fork to keep it that way. Other platforms are
+not tested yet.  The current status is already good enough to
+work on both build related issues (e.g. build warnings, debugging some C/C++
+extension) and on general SciPy development. It is already a much smoother than
+working with the default `distutils`-based build one gets with
+`python setup.py develop`.
+
+The above configure-build-install-test docs are useful to understand and for
+working on build improvements (you don't need to install for that to, for
+example, see the build succeeds and a build warning has disappeared); if you
+want the "all-in-one" command for the above, run:
+```bash
+./mesondev.sh build --prefix=$PWD/installdir
+```
+
+It's worth pointing out that Meson has [very good documentation](https://mesonbuild.com/);
+it's worth reading and is often the best source of answers for "how to do X".
+

default.nix

@@ -0,0 +1,86 @@
+# File is taken from nixos/nixpkgs and adapted.
+
+{ lib
+, fetchFromGitHub
+, python
+, buildPythonPackage
+, gfortran
+, nose
+# Use a hook that implements the checkPhase calling pytest
+# instead of us calling pytest directly.
+, pytestCheckHook
+, pytest-xdist
+, numpy
+, pybind11
+, cython
+, meson
+, ninja
+, pythran
+, pkg-config
+, boost175
+}:
+
+let
+
+  excludeIndices = list: indices: let
+    nitems = lib.length list;
+    includeIndices = lib.subtractLists indices (lib.range 0 (nitems - 1));
+  in map (idx: lib.elemAt list idx) includeIndices;
+
+  meson_ = meson.overrideAttrs (oldAttrs: rec {
+    version = "0.59.0.rc2";
+    src = fetchFromGitHub {
+    owner = "rgommers";
+    repo = "meson";
+    rev = "087870a2f15b31c56c4dc830ae856c3729f60776";
+    sha256 = "sha256-7HJjxVnryaBkRDiwMHVkOo6mahfjrGAf1JijBIZ7Ay4=";
+    };
+    patches = excludeIndices oldAttrs.patches [ 2 ]; # remove gir fallback path patch
+  });
+
+in buildPythonPackage rec {
+  pname = "scipy";
+  version = "dev";
+  format = "other";
+
+  src = ./.;
+
+  checkInputs = [ pytestCheckHook pytest-xdist ];
+  nativeBuildInputs = [ cython gfortran meson_ ninja pythran pkg-config ];
+  buildInputs = [ numpy.blas pybind11 boost175 ];
+  propagatedBuildInputs = [ numpy ];
+
+  # Remove tests because of broken wrapper
+  prePatch = ''
+    rm scipy/linalg/tests/test_lapack.py
+  '';
+
+  doCheck = true;
+
+  preBuild = ''
+    ln -s ${numpy.cfg} site.cfg
+  '';
+
+  passthru = {
+    blas = numpy.blas;
+    meson = meson_;
+  };
+
+  preCheck = ''
+    pushd $out/${python.sitePackages}
+    export PYTHONPATH=$out/${python.sitePackages}:$PYTHONPATH
+  '';
+
+  postCheck = ''
+    popd
+  '';
+
+  SCIPY_USE_G77_ABI_WRAPPER = 1;
+
+  meta = with lib; {
+    description = "SciPy (pronounced 'Sigh Pie') is open-source software for mathematics, science, and engineering";
+    homepage = "https://www.scipy.org/";
+    license = licenses.bsd3;
+    maintainers = [ maintainers.fridh ];
+  };
+}

environment.yml

@@ -10,8 +10,10 @@ dependencies:
   - python
   - cython
   - compilers
+  - ninja
   - numpy
   - openblas
+  - pkg-config
   - libblas=*=*openblas  # helps avoid pulling in MKL
   - pybind11
   - pythran>=0.9.11