diff --git a/.circleci/config.yml b/.circleci/config.yml index 9eb63ece5..4351ad001 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -51,7 +51,6 @@ jobs: command: | python -m pip install --user --upgrade --progress-bar off pip python -m pip install --user -e . - python -m pip install --user --upgrade --no-cache-dir --progress-bar off -r requirements_all.txt python -m pip install --user --upgrade --progress-bar off -r docs/requirements.txt python -m pip install --user --upgrade --progress-bar off ipython sphinx-gallery memory_profiler # python -m pip install --user --upgrade --progress-bar off ipython "https://api.github.com/repos/sphinx-gallery/sphinx-gallery/zipball/master" memory_profiler diff --git a/requirements_all.txt b/.github/requirements_doctests.txt similarity index 67% rename from requirements_all.txt rename to .github/requirements_doctests.txt index 8a1a5f2f5..95000846d 100644 --- a/requirements_all.txt +++ b/.github/requirements_doctests.txt @@ -2,10 +2,10 @@ numpy>=1.20 scipy>=1.6 matplotlib autograd -pymanopt @ git+https://github.com/pymanopt/pymanopt.git@master +pymanopt cvxopt scikit-learn -torch<=2.11 +torch<2.12 jax jaxlib tensorflow; python_version < '3.14' diff --git a/.github/requirements_no_backend.txt b/.github/requirements_no_backend.txt new file mode 100644 index 000000000..6c91630af --- /dev/null +++ b/.github/requirements_no_backend.txt @@ -0,0 +1,9 @@ +numpy>=1.20 +scipy>=1.6 +matplotlib +autograd +pymanopt +cvxopt +scikit-learn +pytest +cvxpy diff --git a/.github/workflows/build_doc.yml b/.github/workflows/build_doc.yml index e1620683f..fcc45fb4b 100644 --- a/.github/workflows/build_doc.yml +++ b/.github/workflows/build_doc.yml @@ -5,7 +5,7 @@ on: pull_request: push: branches: - - 'master' + - 'master' jobs: build: @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: submodules: true # Standard drop-in approach that should work for most people. @@ -31,7 +31,6 @@ jobs: - name: Get Python running run: | python -m pip install --user --upgrade --progress-bar off pip - python -m pip install --user --upgrade --progress-bar off -r requirements_all.txt python -m pip install --user --upgrade --progress-bar off -r docs/requirements.txt python -m pip install --user --upgrade --progress-bar off ipython sphinx-gallery memory_profiler python -m pip install -v --user -e . @@ -45,7 +44,7 @@ jobs: uses: rickstaa/sphinx-action@master with: docs-folder: "docs/" - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 with: name: Documentation path: docs/build/html/ diff --git a/.github/workflows/build_tests.yml b/.github/workflows/build_tests.yml index 2510004c1..df0652a3b 100644 --- a/.github/workflows/build_tests.yml +++ b/.github/workflows/build_tests.yml @@ -14,6 +14,9 @@ on: tags: - '**' +env: + PY_VERSION: "3.13" + jobs: Lint: @@ -27,13 +30,13 @@ jobs: - name: Checking Out Repository - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: submodules: true # Install Python & Packages - uses: actions/setup-python@v6 with: - python-version: "3.14" + python-version: ${{ env.PY_VERSION }} - run: which python - name: Lint with pre-commit run: | @@ -44,14 +47,14 @@ jobs: build_from_source: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: submodules: true - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: - python-version: "3.14" + python-version: ${{ env.PY_VERSION }} - name: Build from source run: | python -m pip install --upgrade pip setuptools wheel @@ -59,22 +62,77 @@ jobs: python setup.py sdist bdist_wheel pip install dist/*.tar.gz - linux: + doctests: runs-on: ubuntu-latest if: "!contains(github.event.head_commit.message, 'no ci')" - strategy: - max-parallel: 4 - matrix: - python-version: ["3.11", "3.12", "3.13", "3.14"] - steps: - name: Free Disk Space (Ubuntu) uses: insightsengineering/disk-space-reclaimer@v1 with: android: true dotnet: true - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 + with: + submodules: true + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: ${{ env.PY_VERSION }} + cache: 'pip' + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools + pip install -r .github/requirements_doctests.txt + pip install pytest pytest-cov + - name: Install POT + run: | + pip install -e . + - name: Run tests + run: | + python -m pytest -v ot/ test/conftest.py --doctest-modules --color=yes --cov=./ --cov-report=xml + + linux-minimal: + + runs-on: ubuntu-latest + if: "!contains(github.event.head_commit.message, 'no ci')" + steps: + - uses: actions/checkout@v7 + with: + submodules: true + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: ${{ env.PY_VERSION }} + cache: 'pip' + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools + pip install pytest pytest-cov + - name: Install POT + run: | + pip install -e . + - name: Run tests + run: | + python -m pytest --durations=20 -v test/ ot/ --color=yes --cov=./ --cov-report=xml + + linux: + runs-on: ubuntu-latest + if: "!contains(github.event.head_commit.message, 'no ci')" + strategy: + max-parallel: 4 + matrix: + python-version: ["3.11", "3.12", "3.13", "3.14"] + + steps: + # - name: Free Disk Space (Ubuntu) + # uses: insightsengineering/disk-space-reclaimer@v1 + # with: + # android: true + # dotnet: true + - uses: actions/checkout@v7 with: submodules: true @@ -89,41 +147,100 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip setuptools - pip install -r requirements_all.txt + pip install -r .github/requirements_no_backend.txt pip install pytest pytest-cov - name: test POT import run: | python -c "import ot; print(ot.__version__)" - name: Run tests run: | - python -m pytest --durations=20 -v test/ ot/ --doctest-modules --color=yes --cov=./ --cov-report=xml + python -m pytest --durations=20 -v test/ --color=yes --cov=./ --cov-report=xml - name: Upload coverage reports to Codecov with GitHub Action uses: codecov/codecov-action@v4 - linux-minimal-deps: + linux-torch: + runs-on: ubuntu-latest + if: "!contains(github.event.head_commit.message, 'no ci')" + + steps: + - uses: actions/checkout@v7 + with: + submodules: true + + - name: Set up Python ${{ env.PY_VERSION }} + uses: actions/setup-python@v6 + with: + python-version: ${{ env.PY_VERSION }} + cache: 'pip' + - name: Install POT + run: | + pip install -e . + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools + pip install torch torch_geometric geomloss pykeops scikit-learn + pip install pytest pytest-cov + - name: Run tests + run: | + python -m pytest --durations=20 -v test/ --color=yes --cov=./ --cov-report=xml + - name: Upload coverage reports to Codecov with GitHub Action + uses: codecov/codecov-action@v4 + linux-jax: runs-on: ubuntu-latest if: "!contains(github.event.head_commit.message, 'no ci')" + steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: submodules: true - - name: Set up Python - uses: actions/setup-python@v5 + - name: Set up Python ${{ env.PY_VERSION }} + uses: actions/setup-python@v6 with: - python-version: "3.14" + python-version: ${{ env.PY_VERSION }} cache: 'pip' + - name: Install POT + run: | + pip install -e . - name: Install dependencies run: | python -m pip install --upgrade pip setuptools + pip install jax jaxlib pip install pytest pytest-cov + - name: Run tests + run: | + python -m pytest --durations=20 -v test/ --color=yes --cov=./ --cov-report=xml + - name: Upload coverage reports to Codecov with GitHub Action + uses: codecov/codecov-action@v4 + + linux-tf: + runs-on: ubuntu-latest + if: "!contains(github.event.head_commit.message, 'no ci')" + + steps: + - uses: actions/checkout@v7 + with: + submodules: true + + - name: Set up Python ${{ env.PY_VERSION }} + uses: actions/setup-python@v6 + with: + python-version: ${{ env.PY_VERSION }} + cache: 'pip' - name: Install POT run: | pip install -e . + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools + pip install tensorflow + pip install pytest pytest-cov - name: Run tests run: | - python -m pytest --durations=20 -v test/ ot/ --color=yes --cov=./ --cov-report=xml + python -m pytest --durations=20 -v test/ --color=yes --cov=./ --cov-report=xml + - name: Upload coverage reports to Codecov with GitHub Action + uses: codecov/codecov-action@v4 # linux-minimal-deps-ft: @@ -151,22 +268,17 @@ jobs: # python -m pytest --durations=20 -v test/ ot/ --color=yes --cov=./ --cov-report=xml macos: - runs-on: ${{ matrix.os }} + runs-on: macos-latest if: "!contains(github.event.head_commit.message, 'no ci')" - strategy: - max-parallel: 4 - matrix: - os: [macos-latest] - python-version: ["3.13"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: submodules: true - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + - name: Set up Python ${{ env.PY_VERSION }} + uses: actions/setup-python@v6 with: - python-version: ${{ matrix.python-version }} + python-version: ${{ env.PY_VERSION }} cache: 'pip' - name: Install POT run: | @@ -174,7 +286,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip setuptools - pip install -r requirements_all.txt + pip install -r .github/requirements_no_backend.txt pip install pytest - name: Run tests run: | @@ -184,19 +296,15 @@ jobs: windows: runs-on: windows-latest if: "!contains(github.event.head_commit.message, 'no ci')" - strategy: - max-parallel: 4 - matrix: - python-version: ["3.13"] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: submodules: true - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + - name: Set up Python ${{ env.PY_VERSION }} + uses: actions/setup-python@v6 with: - python-version: ${{ matrix.python-version }} + python-version: ${{ env.PY_VERSION }} cache: 'pip' - name: RC.exe run: | @@ -213,7 +321,7 @@ jobs: Invoke-VSDevEnvironment Get-Command rc.exe | Format-Table -AutoSize - name: Update pip - run : | + run: | python -m pip install --upgrade pip setuptools python -m pip install cython - name: Install POT diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index e0158d89d..8617eb6bd 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -15,11 +15,11 @@ jobs: os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest, windows-latest] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: submodules: true - - name: Set up Python 3.10 - uses: actions/setup-python@v5 + - name: Set up Python 3.11 + uses: actions/setup-python@v6 with: python-version: "3.11" @@ -41,5 +41,3 @@ jobs: with: name: wheels-${{ strategy.job-index }} path: ./wheelhouse - - diff --git a/.yamllint.yml b/.yamllint.yml index e7c081ac9..5d871be42 100644 --- a/.yamllint.yml +++ b/.yamllint.yml @@ -1,7 +1,6 @@ extends: default ignore: | - .github/workflows/*.yml .circleci/config.yml codecov.yml .github/labeler.yml @@ -9,3 +8,4 @@ ignore: | rules: line-length: disable document-start: disable + indentation: disable diff --git a/CITATION.cff b/CITATION.cff index 2b0c64ac0..2c73bd3bd 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -84,11 +84,12 @@ authors: - given-names: Marco family-names: Corneli affiliation: Université Côte d'Azur - identifiers: - type: url value: 'https://github.com/PythonOT/POT' description: Code + - type: doi + value: 10.5281/zenodo.17161062 repository-code: 'https://github.com/PythonOT/POT' url: 'https://pythonot.github.io/' keywords: @@ -98,4 +99,4 @@ keywords: - wasserstein - gromov-wasserstein license: MIT -version: 0.9.5 +version: 0.9.7 diff --git a/RELEASES.md b/RELEASES.md index d74a11e82..51d89cc01 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,8 +1,8 @@ # Releases -## 0.9.7.dev0## 0.9.7.dev0 +## 0.9.7 -This new release adds support for sparse cost matrices and a new lazy EMD solver that computes distances on-the-fly from coordinates, reducing memory usage from O(n×m) to O(n+m). Both implementations are backend-agnostic and preserve gradient computation for automatic differentiation. +This new release adds support for sparse cost matrices and a new lazy exact OT solver that re-computes distances on-the-fly from coordinates, reducing memory usage from O(n×m) to O(n+m). Both implementations are backend-agnostic and preserve gradient computation for automatic differentiation. #### New features @@ -34,7 +34,8 @@ This new release adds support for sparse cost matrices and a new lazy EMD solver - Wrapper for barycenter solvers with free support `ot.solvers.bary_free_support` (PR #730) - Build wheels on ubuntu ARM to avoid QEMU emulation (PR #818) - Add new methods to compute the linear transport map and the related 2-Wasserstein distance betweeen high-dimensional (HD) Gaussian distributions as described in [88], implemented in `ot.gaussian.bures_wasserstein_mapping_hd` and `ot.gaussian.bures_wasserstein_distance_hd`, respectively. Two additional methods estimate the same quantities from the source and destination observed data and are implemented in `ot.gaussian.empirical_bures_wasserstein_mapping_hd` and `ot.gaussian.empirical_bures_wasserstein_distance_hd`, respectively (PR #814) -- Fix docstrings for `lowrank_gromov_wasserstein_samples` and `lowrank_sinkhorn` (PR #823) +- Fix docstrings for `lowrank_gromov_wasserstein_samples` and `lowrank_sinkhorn` (PR #823) +- Reorganize all tests per backend (PR #828) #### Closed issues diff --git a/docs/requirements.txt b/docs/requirements.txt index cf689f205..beb66b6bc 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,17 @@ +numpy>=1.20 +scipy>=1.6 +matplotlib +autograd +pymanopt +cvxopt +scikit-learn +torch<2.12 +pytest +torch_geometric +cvxpy +geomloss +pykeops + sphinx sphinx-rtd-theme sphinx-gallery diff --git a/ot/gnn/_layers.py b/ot/gnn/_layers.py index b3adad489..1d1b54c59 100644 --- a/ot/gnn/_layers.py +++ b/ot/gnn/_layers.py @@ -7,14 +7,16 @@ # Rémi Flamary # # License: MIT License - -import torch -import torch.nn as nn -from ._utils import ( - TFGW_template_initialization, - FGW_distance_to_templates, - wasserstein_distance_to_templates, -) +try: + import torch + import torch.nn as nn + from ._utils import ( + TFGW_template_initialization, + FGW_distance_to_templates, + wasserstein_distance_to_templates, + ) +except ImportError: + pass class TFGWPooling(nn.Module): diff --git a/ot/helpers/openmp_helpers.py b/ot/helpers/openmp_helpers.py index ae2490542..fbe256625 100644 --- a/ot/helpers/openmp_helpers.py +++ b/ot/helpers/openmp_helpers.py @@ -5,12 +5,73 @@ import os import sys +import glob +import tempfile import textwrap import subprocess from setuptools.errors import CompileError, LinkError +from setuptools.command.build_ext import customize_compiler, new_compiler -from pre_build_helpers import compile_test_program + +def _get_compiler(): + ccompiler = new_compiler() + customize_compiler(ccompiler) + return ccompiler + + +def compile_test_program(code, extra_preargs=[], extra_postargs=[]): + """Check that some C code can be compiled and run""" + ccompiler = _get_compiler() + + # extra_(pre/post)args can be a callable to make it possible to get its + # value from the compiler + if callable(extra_preargs): + extra_preargs = extra_preargs(ccompiler) + if callable(extra_postargs): + extra_postargs = extra_postargs(ccompiler) + + start_dir = os.path.abspath(".") + + with tempfile.TemporaryDirectory() as tmp_dir: + try: + os.chdir(tmp_dir) + + # Write test program + with open("test_program.c", "w") as f: + f.write(code) + + os.mkdir("objects") + + # Compile, test program + ccompiler.compile( + ["test_program.c"], output_dir="objects", extra_postargs=extra_postargs + ) + + # Link test program + objects = glob.glob(os.path.join("objects", "*" + ccompiler.obj_extension)) + ccompiler.link_executable( + objects, + "test_program", + extra_preargs=extra_preargs, + extra_postargs=extra_postargs, + ) + + if "PYTHON_CROSSENV" not in os.environ: + # Run test program if not cross compiling + # will raise a CalledProcessError if return code was non-zero + output = subprocess.check_output("./test_program") + output = output.decode(sys.stdout.encoding or "utf-8").splitlines() + else: + # Return an empty output if we are cross compiling + # as we cannot run the test_program + output = [] + except Exception: + raise + finally: + os.chdir(start_dir) + + return output, extra_postargs def get_openmp_flag(compiler): diff --git a/ot/helpers/pre_build_helpers.py b/ot/helpers/pre_build_helpers.py deleted file mode 100644 index 51b231c42..000000000 --- a/ot/helpers/pre_build_helpers.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Helpers to check build environment before actual build of POT""" - -import os -import sys -import glob -import tempfile -import subprocess - -from setuptools.command.build_ext import customize_compiler, new_compiler - - -def _get_compiler(): - ccompiler = new_compiler() - customize_compiler(ccompiler) - return ccompiler - - -def compile_test_program(code, extra_preargs=[], extra_postargs=[]): - """Check that some C code can be compiled and run""" - ccompiler = _get_compiler() - - # extra_(pre/post)args can be a callable to make it possible to get its - # value from the compiler - if callable(extra_preargs): - extra_preargs = extra_preargs(ccompiler) - if callable(extra_postargs): - extra_postargs = extra_postargs(ccompiler) - - start_dir = os.path.abspath(".") - - with tempfile.TemporaryDirectory() as tmp_dir: - try: - os.chdir(tmp_dir) - - # Write test program - with open("test_program.c", "w") as f: - f.write(code) - - os.mkdir("objects") - - # Compile, test program - ccompiler.compile( - ["test_program.c"], output_dir="objects", extra_postargs=extra_postargs - ) - - # Link test program - objects = glob.glob(os.path.join("objects", "*" + ccompiler.obj_extension)) - ccompiler.link_executable( - objects, - "test_program", - extra_preargs=extra_preargs, - extra_postargs=extra_postargs, - ) - - if "PYTHON_CROSSENV" not in os.environ: - # Run test program if not cross compiling - # will raise a CalledProcessError if return code was non-zero - output = subprocess.check_output("./test_program") - output = output.decode(sys.stdout.encoding or "utf-8").splitlines() - else: - # Return an empty output if we are cross compiling - # as we cannot run the test_program - output = [] - except Exception: - raise - finally: - os.chdir(start_dir) - - return output, extra_postargs diff --git a/setup.py b/setup.py index 4cfde74d4..f423f1c0e 100644 --- a/setup.py +++ b/setup.py @@ -15,19 +15,20 @@ import numpy from Cython.Build import cythonize -sys.path.append(os.path.join("ot", "helpers")) +ROOT = os.path.abspath(os.path.dirname(__file__)) + +sys.path.append(os.path.join(ROOT, "ot", "helpers")) from openmp_helpers import check_openmp_support # dirty but working __version__ = re.search( r'__version__\s*=\s*[\'"]([^\'"]*)[\'"]', # It excludes inline comment too - open("ot/__init__.py").read(), + open(os.path.join(ROOT, "ot/__init__.py")).read(), ).group(1) # The beautiful part is, I don't even need to check exceptions here. # If something messes up, let the build process fail noisy, BEFORE my release! # thanks PyPI for handling markdown now -ROOT = os.path.abspath(os.path.dirname(__file__)) with open(os.path.join(ROOT, "README.md"), encoding="utf-8") as f: README = f.read() diff --git a/test/test_helpers.py b/test/test_helpers.py index 7a605f46f..28d0722ba 100644 --- a/test/test_helpers.py +++ b/test/test_helpers.py @@ -9,8 +9,12 @@ sys.path.append(os.path.join("ot", "helpers")) -from openmp_helpers import get_openmp_flag, check_openmp_support # noqa -from pre_build_helpers import _get_compiler, compile_test_program # noqa +from openmp_helpers import ( + get_openmp_flag, + check_openmp_support, + _get_compiler, + compile_test_program, +) # noqa def test_helpers():