test: make tests idempotent and CI-compatible

Author: d-laub

pixi.lock

@@ -38,12 +38,14 @@ patchelf = "*"
 commitizen = "*"
 typer = "*"
 uv = "*"
+samtools = "*"
+bcftools = "*"
 
 [pypi-dependencies]
 genvarloader = { path = ".", editable = true }
-seqpro = "==0.3.1"
 hirola = "==0.3"
-genoray = "==0.10.1"
+seqpro = "==0.3.2"
+genoray = "==0.10.2"
 
 [feature.docs.dependencies]
 sphinx = ">=7.4.7"
@@ -79,21 +81,21 @@ python = "3.12.*"
 
 [environments]
 dev = { features = ["pytorch-cpu", "basenji2", "py310"] }
-docs = { features = ["docs", "pytorch-cpu", "basenji2", "py310"] }
+docs = { features = ["docs", "pytorch-cpu", "basenji2", "py312"] }
 py310 = { features = ["pytorch-cpu", "py310"] }
 py311 = { features = ["pytorch-cpu", "py311"] }
 py312 = { features = ["pytorch-cpu", "py312"] }
 # docs-gpu = { features = ["docs", "pytorch-gpu", "basenji2", "py310"] }
 
 [tasks]
-install = "uv pip install -e /cellar/users/dlaub/projects/genoray -e ."
+install = "uv pip install -e /cellar/users/dlaub/projects/ML4GLand/SeqPro -e /cellar/users/dlaub/projects/genoray -e ."
 pre-commit = "pre-commit install --hook-type commit-msg"
 gen = "python tests/data/generate_ground_truth.py"
 test = { cmd = "pytest tests && cargo test --release", depends-on = ["gen"] }
 publish = "maturin publish"
 
 [feature.docs.tasks]
-install = "uv pip install -e /cellar/users/dlaub/projects/genoray -e ."
+install = "uv pip install -e /cellar/users/dlaub/projects/ML4GLand/SeqPro -e /cellar/users/dlaub/projects/genoray -e ."
 i-kernel = "ipython kernel install --user --name 'gvl-docs' --display-name 'GVL Docs'"
 i-kernel-gpu = "ipython kernel install --user --name 'gvl-docs-gpu' --display-name 'GVL Docs GPU'"
 doc = "cd docs && make clean && make html"

pixi.toml

@@ -27,10 +27,10 @@ dependencies = [
     "tbb",
     "joblib",
     "pooch",
-    "seqpro>=0.3.1",
     "awkward",
     "hirola>=0.3,<0.4",
-    "genoray>=0.10.1",
+    "seqpro>=0.3.2",
+    "genoray>=0.10.2",
 ]
 
 [project.urls]
@@ -54,6 +54,7 @@ filterwarnings = [
     "ignore::DeprecationWarning:ray.*",
     "ignore::DeprecationWarning:pkg_resources.*",
     "ignore::DeprecationWarning:jupyter_client.*",
+    "ignore::DeprecationWarning:sorted_nearest.*",
 ]
 markers = ["slow: mark test as slow (deselect with '-m \"not slow\"')"]
 

pyproject.toml

@@ -594,7 +594,7 @@ def _write_from_svar(
         )
         # this is fine if there aren't any overlapping variants that could make a v_idx < -1
         # have a further end than v_idx == -1
-        #! calling ak.max() means v_idxs is not a view of svar.genos.data
+        #* calling ak.max() means v_idxs is not a view of svar.genos.data
         # (r s p ~v) -> (r)
         v_idxs = ak.max(sp_genos.to_awkward(), -1).to_numpy().max((1, 2))  # type: ignore
         c_max_ends = max_ends[contig_offset : contig_offset + df.height]

python/genvarloader/_dataset/_write.py

@@ -3,8 +3,8 @@
 ##fileDate=20230929
 ##source=myImputationProgramV3.1
 ##reference=Homo_sapiens.GRCh38.p13_v109.dna.toplevel.fa.gz
-##contig=<ID=19>
-##contig=<ID=20>
+##contig=<ID=chr19>
+##contig=<ID=chr20>
 ##phasing=partial
 ##INFO=<ID=NS,Number=1,Type=Integer,Description="Number of Samples With Data">
 ##INFO=<ID=AN,Number=1,Type=Integer,Description="Total number of alleles in called genotypes">
@@ -24,17 +24,17 @@
 ##ALT=<ID=DEL:ME:ALU,Description="Deletion of ALU element">
 ##ALT=<ID=CNV,Description="Copy number variable region">
 #CHROM	POS	ID	REF	ALT	QUAL	FILTER	INFO	FORMAT	NA00001	NA00002	NA00003
-19	111	.	N	C	9.6	.	.	GT:VAF:HQ	0|0:.:10,15	0|0:.:10,10	0/1:.3:3,3
-19	1010696	.	GAGA	G	10	.	.	GT:VAF:HQ	1|0:.3:10,10	0|0:.1:10,15	0/0:.:3,3
-19	1010696	.	GAGACGG	G	10	.	.	GT:VAF:HQ	0|0:.:10,10	0|0:.5:10,15	0/1:.15:3,3
-19	1010696	.	GAGACGGGGCC	G	10	.	.	GT:VAF:HQ	0|1:.2:10,10	1|1:.6:10,15	0/0:.:3,3
-19	1110696	.	A	TTT	10	.	.	GT:VAF:HQ	0|1:.25:10,10	1|1:.45:10,15	0/0:.:3,3
-19	1110696	.	A	G	10	.	.	GT:VAF:HQ	0|0:.:10,10	0|0:.:10,10	0/1:.2:3,3
-19	1210696	.	C	G	10	.	.	GT:VAF:HQ	1|.:0.1:10,10	0/1:0.23:10,10	1|1:0.2:3,3
-19	1210696	.	C	G	10	.	.	GT:VAF:HQ	.|1:0.15:10,10	0|0:.:10,10	0/0:.:3,3
-19	1210697	.	T	G	10	.	.	GT:VAF:HQ	0/0:.:10,10	1|0:0.05:10,10	0/1:0.2:3,3
-19	1210697	.	T	A	10	.	.	GT:VAF:HQ	0/0:.:10,10	1|0:0.4:10,10	0/1:0.3:3,3
-20	14370	rs6054257	N	A	29	PASS	NS=3;DP=14;AF=0.5;DB;H2	GT:VAF:GQ:DP:HQ	0|0:.:48:1:51,51	1|0:.15:48:8:51,51	1/1:.65:43:5:.,.
-20	17330	.	N	A	3	q10	NS=3;DP=11;AF=0.017	GT:VAF:GQ:DP:HQ	0|0:.:49:3:58,50	0|1:.3:3:5:65,3	0/0:.:41:3:.,.
-20	1110696	rs6040355	G	A,T	67	PASS	NS=2;DP=10;AF=0.333,0.667;AA=T;DB	GT:VAF:GQ:DP:HQ	1|2:.55:21:6:23,27	2|1:.45:2:0:18,2	2/2:.6:35:4:.,.
-20	1234567	microsat1	A	GA,AC	50	PASS	NS=3;DP=9;AA=G;AN=6;AC=3,1	GT:VAF:GQ:DP	0/1:.2:.:4	0/2:.15:17:2	./.:.:40:3
+chr19	111	.	N	C	9.6	.	.	GT:VAF:HQ	0|0:.:10,15	0|0:.:10,10	0/1:.3:3,3
+chr19	1010696	.	GAGA	G	10	.	.	GT:VAF:HQ	1|0:.3:10,10	0|0:.1:10,15	0/0:.:3,3
+chr19	1010696	.	GAGACGG	G	10	.	.	GT:VAF:HQ	0|0:.:10,10	0|0:.5:10,15	0/1:.15:3,3
+chr19	1010696	.	GAGACGGGGCC	G	10	.	.	GT:VAF:HQ	0|1:.2:10,10	1|1:.6:10,15	0/0:.:3,3
+chr19	1110696	.	A	TTT	10	.	.	GT:VAF:HQ	0|1:.25:10,10	1|1:.45:10,15	0/0:.:3,3
+chr19	1110696	.	A	G	10	.	.	GT:VAF:HQ	0|0:.:10,10	0|0:.:10,10	0/1:.2:3,3
+chr19	1210696	.	C	G	10	.	.	GT:VAF:HQ	1|.:0.1:10,10	0/1:0.23:10,10	1|1:0.2:3,3
+chr19	1210696	.	C	G	10	.	.	GT:VAF:HQ	.|1:0.15:10,10	0|0:.:10,10	0/0:.:3,3
+chr19	1210697	.	T	G	10	.	.	GT:VAF:HQ	0/0:.:10,10	1|0:0.05:10,10	0/1:0.2:3,3
+chr19	1210697	.	T	A	10	.	.	GT:VAF:HQ	0/0:.:10,10	1|0:0.4:10,10	0/1:0.3:3,3
+chr20	14370	rs6054257	N	A	29	PASS	NS=3;DP=14;AF=0.5;DB;H2	GT:VAF:GQ:DP:HQ	0|0:.:48:1:51,51	1|0:.15:48:8:51,51	1/1:.65:43:5:.,.
+chr20	17330	.	N	A	3	q10	NS=3;DP=11;AF=0.017	GT:VAF:GQ:DP:HQ	0|0:.:49:3:58,50	0|1:.3:3:5:65,3	0/0:.:41:3:.,.
+chr20	1110696	rs6040355	G	A,T	67	PASS	NS=2;DP=10;AF=0.333,0.667;AA=T;DB	GT:VAF:GQ:DP:HQ	1|2:.55:21:6:23,27	2|1:.45:2:0:18,2	2/2:.6:35:4:.,.
+chr20	1234567	microsat1	A	GA,AC	50	PASS	NS=3;DP=9;AA=G;AN=6;AC=3,1	GT:VAF:GQ:DP	0/1:.2:.:4	0/2:.15:17:2	./.:.:40:3

tests/data/source.vcf

@@ -5,7 +5,7 @@
 from pytest_cases import fixture, parametrize_with_cases
 
 data_dir = Path(__file__).resolve().parents[1] / "data"
-ref = data_dir / "fasta" / "Homo_sapiens.GRCh38.dna.primary_assembly.fa.bgz"
+ref = data_dir / "fasta" / "hg38.fa.bgz"
 
 def ds_phased():
     return gvl.Dataset.open(data_dir / "phased_dataset.vcf.gvl", ref)

tests/dataset/test_dataset.py

@@ -8,7 +8,7 @@
 from pytest_cases import parametrize_with_cases
 
 data_dir = Path(__file__).resolve().parents[1] / "data"
-ref = data_dir / "fasta" / "Homo_sapiens.GRCh38.dna.primary_assembly.fa.bgz"
+ref = data_dir / "fasta" / "hg38.fa.bgz"
 cons_dir = data_dir / "consensus"
 
 
@@ -40,7 +40,7 @@ def test_ds_haps(dataset: gvl.RaggedDataset[gvl.Ragged[np.bytes_], None, None, N
             actual = haps[h]
             fpath = f"source_{sample}_nr{region}_h{h}.fa"
             with pysam.FastaFile(str(cons_dir / fpath)) as f:
-                desired = sp.cast_seqs(f.fetch(f.references[0]))
+                desired = sp.cast_seqs(f.fetch(f.references[0]).upper())
             np.testing.assert_equal(
                 actual,
                 desired,

tests/dataset/test_ds_haps.py

@@ -7,7 +7,7 @@
 from pytest_cases import parametrize_with_cases
 
 DATA_DIR = Path(__file__).resolve().parents[1] / "data"
-REF = DATA_DIR / "fasta" / "Homo_sapiens.GRCh38.dna.primary_assembly.fa.bgz"
+REF = DATA_DIR / "fasta" / "hg38.fa.bgz"
 DATASET = gvl.Dataset.open(DATA_DIR / "phased_dataset.vcf.gvl", REF)
 
 

tests/dataset/test_subset.py

@@ -36,7 +36,7 @@ def bed():
 
 @fixture
 def ref():
-    return ddir / "fasta" / "Homo_sapiens.GRCh38.dna.primary_assembly.fa.bgz"
+    return ddir / "fasta" / "hg38.fa.bgz"
 
 
 @mark.skip
@@ -101,4 +101,4 @@ def test_write(reader: Reader, bed: pl.DataFrame, ref: Path, tmp_path):
     max_len = lengths.max()
     for len_ in range(1, max_len + 1):
         mask = ak.num(desired, -1) == len_
-        assert ak.all(actual[mask][:, :len_] == desired[mask][:, :len_])
+        assert ak.all(actual[mask][:, :len_] == desired[mask][:, :len_])  # type: ignore

tests/dataset/test_write.py

@@ -13,13 +13,13 @@ def fasta_path():
         Path(__file__).parent
         / "data"
         / "fasta"
-        / "Homo_sapiens.GRCh38.dna.primary_assembly.fa.bgz"
+        / "hg38.fa.bgz"
     )
 
 
 def test_pad_right(fasta_path):
     fasta = Fasta("ref", fasta_path, pad="N")
-    contig = "1"
+    contig = "chr1"
     with FastaFile(fasta_path) as f:
         end_of_contig = f.get_reference_length(contig)
         start = end_of_contig - 5
@@ -36,7 +36,7 @@ def test_pad_right(fasta_path):
 
 def test_pad_left(fasta_path):
     fasta = Fasta("ref", fasta_path, pad="N")
-    contig = "1"
+    contig = "chr1"
     start = -5
     end = start + 10
     seq = fasta.read(contig, start, end)
@@ -50,13 +50,13 @@ def test_pad_left(fasta_path):
 def test_no_pad(fasta_path):
     fasta = Fasta("ref", fasta_path)
     end_of_contig_1 = 248956422
-    contig = "1"
+    contig = "chr1"
     start = end_of_contig_1 - 5
     end = start + 10
     with pytest.raises(NoPadError):
         fasta.read(contig, start, end)
 
-    contig = "1"
+    contig = "chr1"
     start = -5
     end = start + 10
     with pytest.raises(NoPadError):