unbreak CI

CHANGELOG.md

@@ -37,6 +37,23 @@
 * Update scPRINT to use latest stable version (PR #70)
 * Fix kbet dependencies to numpy<2 and scipy<=1.13 (PR #78).
 * Fix `render_readme` crashing on `comp_process_integration.yaml`'s absolute `__merge__` paths.
+* Fix `metrics/kbet_pg` and `metrics/kbet_pg_label` failing to build: drop the zarr, pandas and numpy bounds pegasuspy
+    doesn't ask for, and pin `setuptools<81` so `pkg_resources` is still there.
+* Fix `methods/harmonypy` writing a transposed embedding: harmonypy 2.0 returns `Z_corr` as cells x pcs.
+* Fix `methods/stacas` erroring on `GetAssayData(slot = )`, defunct since SeuratObject 5.0.0. Tracks STACAS master
+    until upstream cuts a release containing the fix.
+* Fix `methods/scalex` failing to build: its numpy and torch bounds contradicted what scalex asks for. Keeps
+    `numpy<2`, since scalex still calls `np.Inf`.
+* Fix `methods/pyliger` failing to build: louvain has no python 3.12 wheel and needs cmake to build igraph from source.
+* Bump `methods/cellplm`, `methods/condo`, `methods/drvi` and `metrics/bras` from base image `:1.0.0` to `:1`, so their
+    `openproblems` is new enough for the component tests in `common`.
+* Fix `methods/geneformer` failing to build: pip's `--filter=blob:none` clone of the huggingface repo no longer works,
+    so clone it ourselves. Also needs `transformers<5`, which still has `SpecialTokensMixin`.
+* Fix `methods/cellplm` failing to build: drop the pytorch base image's broken `/usr/local/bin/cmake` shim, which
+    shadowed the apt cmake that louvain needs.
+* Fix `methods/scgpt_zeroshot` and `methods/scgpt_finetuned` failing to build: stop installing `flash-attn`. Both
+    scripts pass `use_fast_transformer=False`, so it was never used. Behind it sat three more pins with no python 3.12
+    wheels, now `numpy<2`, `torchtext==0.17.2` and `transformers==4.36.2`.
 
 * Split Scanorama into two methods/scores
     - Split Scanorama into embedding (integrate) and count-correction (correct) modes, instead of running both together. 

src/methods/cellplm/config.vsh.yaml

@@ -38,8 +38,12 @@ resources:
 
 engines:
   - type: docker
-    image: openproblems/base_pytorch_nvidia:1.0.0
+    image: openproblems/base_pytorch_nvidia:1
     setup:
+      - type: apt
+        packages: [cmake]
+      - type: docker
+        run: rm -f /usr/local/bin/cmake
       - type: python
         pypi:
           - cellplm

src/methods/condo/config.vsh.yaml

@@ -100,7 +100,7 @@ resources:
   - path: /src/utils/read_anndata_partial.py
 engines:
   - type: docker
-    image: openproblems/base_pytorch_nvidia:1.0.0
+    image: openproblems/base_pytorch_nvidia:1
     setup:
       - type: python
         pypi:

src/methods/drvi/config.vsh.yaml

@@ -30,13 +30,11 @@ resources:
   - path: /src/utils/read_anndata_partial.py
 engines:
   - type: docker
-    image: openproblems/base_pytorch_nvidia:1.0.0 
+    image: openproblems/base_pytorch_nvidia:1
     setup:
       - type: python
         pypi:
-        - drvi-py==0.1.7
-        - torch==2.3.0
-        - torchvision==0.18.0
+        - drvi-py==0.2.7
 runners:
   - type: executable
   - type: nextflow

src/methods/geneformer/config.vsh.yaml

@@ -54,7 +54,10 @@ engines:
         pip:
           - pyarrow<15.0.0a0,>=14.0.1
           - huggingface_hub
-          - git+https://huggingface.co/ctheodoris/Geneformer.git
+      - type: docker
+        run: |
+          git clone --depth 1 https://huggingface.co/ctheodoris/Geneformer.git /opt/Geneformer && \
+            pip install --no-cache-dir /opt/Geneformer "transformers<5"
 
 runners:
   - type: executable

src/methods/harmonypy/config.vsh.yaml

@@ -26,7 +26,7 @@ engines:
     setup:
       - type: python
         pypi:
-          - harmonypy
+          - harmonypy>=2
 runners:
   - type: executable
   - type: nextflow

src/methods/harmonypy/script.py

@@ -38,7 +38,7 @@
     obs=adata.obs[[]],
     var=adata.var[[]],
     obsm={
-        "X_emb": out.Z_corr.transpose()
+        "X_emb": out.Z_corr
     },
     shape=adata.shape,
     uns={

src/methods/pyliger/config.vsh.yaml

@@ -28,6 +28,8 @@ engines:
   - type: docker
     image: openproblems/base_python:1
     setup:
+      - type: apt
+        packages: [cmake]
       - type: python
         pypi:
           - pyliger

src/methods/scalex/config.vsh.yaml

@@ -28,8 +28,7 @@ engines:
       - type: python
         pypi:
           - scalex
-          - numpy<1.24
-          - torch<2.1
+          - numpy<2
 runners:
   - type: executable
   - type: nextflow

src/methods/scgpt_finetuned/config.vsh.yaml

@@ -49,21 +49,12 @@ resources:
 engines:
   - type: docker
     image: openproblems/base_pytorch_nvidia:1
-    # TODO: Try to find working installation of flash attention (flash-attn<1.0.5)
     setup:
-      #- type: python
-      #  pypi:
-      #    - gdown
-      #    - scgpt # Install from PyPI to get dependencies
-      #- type: docker
-      #  # Force re-installing from GitHub to get bug fixes
-      #  run: pip install --upgrade --no-deps --force-reinstall git+https://github.com/bowang-lab/scGPT.git
       - type: docker
         run: |
           git clone https://github.com/bowang-lab/scGPT && \
           pip install torch==2.2.0 torchvision==0.17.0 torchaudio==2.2.0 --index-url https://download.pytorch.org/whl/cu121 && \
-          pip install "flash-attn<1.0.5" --no-build-isolation && \
-          pip install ipykernel pandas scanpy numba "numpy<1.24" torchtext==0.17.0 scib "scvi-tools<1.0" datasets==2.14.5 transformers==4.33.2 wandb "cell-gears<0.0.3" torch_geometric pyarrow==15.0.0 gdown && \
+          pip install ipykernel pandas scanpy numba "numpy<2" torchtext==0.17.2 scib "scvi-tools<1.0" datasets==2.14.5 transformers==4.36.2 wandb "cell-gears<0.0.3" torch_geometric pyarrow==15.0.0 gdown && \
           cd scGPT && pip install -e . --no-deps
 
 runners:

src/methods/scgpt_finetuned/script.py

@@ -227,7 +227,7 @@
     "dropout": 0.2,
     "schedule_ratio": 0.9,  # Learning rate decay
     "log_interval": 100,
-    "fast_transformer": False,  # TODO: Set True if flash-attn is installed
+    "fast_transformer": False,  # requires flash-attn
     "pre_norm": False,
     "amp": True,  # Automatic Mixed Precision
 }

src/methods/scgpt_zeroshot/config.vsh.yaml

@@ -51,21 +51,12 @@ resources:
 engines:
   - type: docker
     image: openproblems/base_pytorch_nvidia:1
-    # TODO: Try to find working installation of flash attention (flash-attn<1.0.5)
     setup:
-      #- type: python
-      #  pypi:
-      #    - gdown
-      #    - scgpt # Install from PyPI to get dependencies
-      #- type: docker
-      #  # Force re-installing from GitHub to get bug fixes
-      #  run: pip install --upgrade --no-deps --force-reinstall git+https://github.com/bowang-lab/scGPT.git
       - type: docker
         run: |
           git clone https://github.com/bowang-lab/scGPT && \
           pip install torch==2.2.0 torchvision==0.17.0 torchaudio==2.2.0 --index-url https://download.pytorch.org/whl/cu121 && \
-          pip install "flash-attn<1.0.5" --no-build-isolation && \
-          pip install ipykernel pandas scanpy numba "numpy<1.24" torchtext==0.17.0 scib "scvi-tools<1.0" datasets==2.14.5 transformers==4.33.2 wandb "cell-gears<0.0.3" torch_geometric pyarrow==15.0.0 gdown && \
+          pip install ipykernel pandas scanpy numba "numpy<2" torchtext==0.17.2 scib "scvi-tools<1.0" datasets==2.14.5 transformers==4.36.2 wandb "cell-gears<0.0.3" torch_geometric pyarrow==15.0.0 gdown && \
           cd scGPT && pip install -e . --no-deps
 
 runners:

src/methods/stacas/config.vsh.yaml

@@ -29,7 +29,7 @@ engines:
     image: openproblems/base_r:1
     setup:
       - type: r
-        github: carmonalab/STACAS@2.3.0
+        github: carmonalab/STACAS
 runners:
   - type: executable
   - type: nextflow

src/metrics/bras/config.vsh.yaml

@@ -27,7 +27,7 @@ resources:
 
 engines:
   - type: docker
-    image: openproblems/base_python:1.0.0
+    image: openproblems/base_python:1
     setup:
       - type: python
         pypi:

src/metrics/kbet_pg/config.vsh.yaml

@@ -39,9 +39,7 @@ engines:
       - type: python
         pypi:
           - pegasuspy
-          - zarr<3.0
-          - pandas<2.0
-          - numpy<2.0
+          - setuptools<81
 runners:
   - type: executable
   - type: nextflow

src/metrics/kbet_pg_label/config.vsh.yaml

@@ -39,9 +39,7 @@ engines:
       - type: python
         pypi:
           - pegasuspy
-          - zarr<3.0
-          - pandas<2.0
-          - numpy<2.0
+          - setuptools<81
 runners:
   - type: executable
   - type: nextflow