Merge remote-tracking branch 'origin/main' into spanorm

Author: rcannood

_viash.yaml

@@ -72,8 +72,3 @@ repositories:
     type: github
     repo: openproblems-bio/openproblems
     tag: build/main
-  - name: core
-    type: github
-    repo: openproblems-bio/core
-    tag: build/main
-    path: viash/core

common

@@ -17,12 +17,30 @@ param_list:
   - id: "mouse_brain_combined/rep1"
     input_sp: "$input_dir/10x_xenium/2023_10x_mouse_brain_xenium/rep1/dataset.zarr"
     input_sc: "$input_dir/allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2/dataset.h5ad"
+    dataset_name: "Mouse brain combined 2023 tenx Xenium replicate 1 2023 Yao scRNAseq"
+    dataset_url: "https://www.10xgenomics.com/datasets/fresh-frozen-mouse-brain-replicates-1-standard;https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE246717"
+    dataset_reference: "https://www.10xgenomics.com/datasets/fresh-frozen-mouse-brain-replicates-1-standard;10.1038/s41586-023-06812-z"
+    dataset_summary: "Demonstration of gene expression profiling for fresh frozen mouse brain on the Xenium platform using the pre-designed Mouse Brain Gene Expression Panel (v1);A high-resolution scRNAseq atlas of cell types in the whole mouse brain"
+    dataset_description: "Demonstration of gene expression profiling for fresh frozen mouse brain on the Xenium platform using the pre-designed Mouse Brain Gene Expression Panel (v1). Replicate results demonstrate the high reproducibility of data generated by the platform. 10x Genomics obtained tissue from a C57BL/6 mouse from Charles River Laboratories. Three adjacent 10µm sections were placed on the same slide. Tissues were prepared following the demonstrated protocols Xenium In Situ for Fresh Frozen Tissues - Tissue Preparation Guide (CG000579) and Xenium In Situ for Fresh Frozen Tissues - Fixation & Permeabilization (CG000581).;See dataset_reference for more information. Note that we only took the 10xv2 data from the dataset."
+    dataset_organism: "mus_musculus"
   - id: "mouse_brain_combined/rep2"
     input_sp: "$input_dir/10x_xenium/2023_10x_mouse_brain_xenium/rep2/dataset.zarr"
     input_sc: "$input_dir/allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2/dataset.h5ad"
+    dataset_name: "Mouse brain combined 2023 tenx Xenium replicate 2 2023 Yao scRNAseq"
+    dataset_url: "https://www.10xgenomics.com/datasets/fresh-frozen-mouse-brain-replicates-1-standard;https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE246717"
+    dataset_reference: "https://www.10xgenomics.com/datasets/fresh-frozen-mouse-brain-replicates-1-standard;10.1038/s41586-023-06812-z"
+    dataset_summary: "Demonstration of gene expression profiling for fresh frozen mouse brain on the Xenium platform using the pre-designed Mouse Brain Gene Expression Panel (v1);A high-resolution scRNAseq atlas of cell types in the whole mouse brain"
+    dataset_description: "Demonstration of gene expression profiling for fresh frozen mouse brain on the Xenium platform using the pre-designed Mouse Brain Gene Expression Panel (v1). Replicate results demonstrate the high reproducibility of data generated by the platform. 10x Genomics obtained tissue from a C57BL/6 mouse from Charles River Laboratories. Three adjacent 10µm sections were placed on the same slide. Tissues were prepared following the demonstrated protocols Xenium In Situ for Fresh Frozen Tissues - Tissue Preparation Guide (CG000579) and Xenium In Situ for Fresh Frozen Tissues - Fixation & Permeabilization (CG000581).;See dataset_reference for more information. Note that we only took the 10xv2 data from the dataset."
+    dataset_organism: "mus_musculus"
   - id: "mouse_brain_combined/rep3"
     input_sp: "$input_dir/10x_xenium/2023_10x_mouse_brain_xenium/rep3/dataset.zarr"
     input_sc: "$input_dir/allen_brain_cell_atlas/2023_yao_mouse_brain_scrnaseq_10xv2/dataset.h5ad"
+    dataset_name: "Mouse brain combined 2023 tenx Xenium replicate 3 2023 Yao scRNAseq"
+    dataset_url: "https://www.10xgenomics.com/datasets/fresh-frozen-mouse-brain-replicates-1-standard;https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE246717"
+    dataset_reference: "https://www.10xgenomics.com/datasets/fresh-frozen-mouse-brain-replicates-1-standard;10.1038/s41586-023-06812-z"
+    dataset_summary: "Demonstration of gene expression profiling for fresh frozen mouse brain on the Xenium platform using the pre-designed Mouse Brain Gene Expression Panel (v1);A high-resolution scRNAseq atlas of cell types in the whole mouse brain"
+    dataset_description: "Demonstration of gene expression profiling for fresh frozen mouse brain on the Xenium platform using the pre-designed Mouse Brain Gene Expression Panel (v1). Replicate results demonstrate the high reproducibility of data generated by the platform. 10x Genomics obtained tissue from a C57BL/6 mouse from Charles River Laboratories. Three adjacent 10µm sections were placed on the same slide. Tissues were prepared following the demonstrated protocols Xenium In Situ for Fresh Frozen Tissues - Tissue Preparation Guide (CG000579) and Xenium In Situ for Fresh Frozen Tissues - Fixation & Permeabilization (CG000581).;See dataset_reference for more information. Note that we only took the 10xv2 data from the dataset."
+    dataset_organism: "mus_musculus"
 
 output_sc: "\$id/output_sc.h5ad"
 output_sp: "\$id/output_sp.zarr"

scripts/create_resources/process_datasets.sh

@@ -14,8 +14,7 @@ cat > /tmp/params.yaml << HERE
 param_list:
 
   - id: "vizgen_merscope/2022_vizgen_human_breast_cancer_merfish/rep1"
-    gcloud_bucket: "vz-ffpe-showcase"
-    dataset_bucket_name: "HumanBreastCancerPatient1"
+    input: "gs://vz-ffpe-showcase/HumanBreastCancerPatient1"
     dataset_name: "Vizgen Human Breast Cancer MERFISH Patient1"
     dataset_url: "https://info.vizgen.com/ffpe-showcase?submissionGuid=a93dbab5-c128-4269-afe3-82ea2bf9cdaf"
     dataset_summary: "Human Breast Cancer data from the MERSCOPE FFPE Human Immuno-Oncology Data Release."
@@ -24,8 +23,7 @@ param_list:
     segmentation_id: ["cell"]
 
   - id: "vizgen_merscope/2022_vizgen_human_liver_cancer_merfish/rep1"
-    gcloud_bucket: "vz-ffpe-showcase"
-    dataset_bucket_name: "HumanLiverCancerPatient1"
+    input: "gs://vz-ffpe-showcase/HumanLiverCancerPatient1"
     dataset_name: "Vizgen Human Liver Cancer MERFISH Patient1"
     dataset_url: "https://info.vizgen.com/ffpe-showcase?submissionGuid=a93dbab5-c128-4269-afe3-82ea2bf9cdaf"
     dataset_summary: "Human Liver Cancer data from the MERSCOPE FFPE Human Immuno-Oncology Data Release."
@@ -34,8 +32,7 @@ param_list:
     segmentation_id: ["cell"]
 
   - id: "vizgen_merscope/2022_vizgen_human_liver_cancer_merfish/rep2"
-    gcloud_bucket: "vz-ffpe-showcase"
-    dataset_bucket_name: "HumanLiverCancerPatient2"
+    input: "gs://vz-ffpe-showcase/HumanLiverCancerPatient2"
     dataset_name: "Vizgen Human Liver Cancer MERFISH Patient2"
     dataset_url: "https://info.vizgen.com/ffpe-showcase?submissionGuid=a93dbab5-c128-4269-afe3-82ea2bf9cdaf"
     dataset_summary: "Human Liver Cancer data from the MERSCOPE FFPE Human Immuno-Oncology Data Release."
@@ -44,8 +41,7 @@ param_list:
     segmentation_id: ["cell"]
 
   - id: "vizgen_merscope/2022_vizgen_human_lung_cancer_merfish/rep1"
-    gcloud_bucket: "vz-ffpe-showcase"
-    dataset_bucket_name: "HumanLungCancerPatient1"
+    input: "gs://vz-ffpe-showcase/HumanLungCancerPatient1"
     dataset_name: "Vizgen Human Lung Cancer MERFISH Patient1"
     dataset_url: "https://info.vizgen.com/ffpe-showcase?submissionGuid=a93dbab5-c128-4269-afe3-82ea2bf9cdaf"
     dataset_summary: "Human Lung Cancer data from the MERSCOPE FFPE Human Immuno-Oncology Data Release."
@@ -54,8 +50,7 @@ param_list:
     segmentation_id: ["cell"]
 
   - id: "vizgen_merscope/2022_vizgen_human_lung_cancer_merfish/rep2"
-    gcloud_bucket: "vz-ffpe-showcase"
-    dataset_bucket_name: "HumanLungCancerPatient2"
+    input: "gs://vz-ffpe-showcase/HumanLungCancerPatient2"
     dataset_name: "Vizgen Human Lung Cancer MERFISH Patient2"
     dataset_url: "https://info.vizgen.com/ffpe-showcase?submissionGuid=a93dbab5-c128-4269-afe3-82ea2bf9cdaf"
     dataset_summary: "Human Lung Cancer data from the MERSCOPE FFPE Human Immuno-Oncology Data Release."
@@ -78,14 +73,11 @@ tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \
   --config common/nextflow_helpers/labels_tw.config \
   --labels datasets,vizgen_merscope
 
-
-
 # More datasets that can be simply added:
 # TODO: Make a decision on replicate naming (see ovarian cancer replicate that has multiple slices)
 
 #   - id: "vizgen_merscope/2022_vizgen_human_colon_cancer_merfish/rep1"
-#     gcloud_bucket: "vz-ffpe-showcase"
-#     dataset_bucket_name: "HumanColonCancerPatient1"
+#     input: "gs://vz-ffpe-showcase/HumanColonCancerPatient1"
 #     dataset_name: "2022 Vizgen Human Colon Cancer MERFISH Patient1"
 #     dataset_url: "https://info.vizgen.com/ffpe-showcase"
 #     dataset_summary: "Human Colon Cancer data from the MERSCOPE FFPE Human Immuno-Oncology Data Release."
@@ -94,8 +86,7 @@ tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \
 #     segmentation_id: ["cell"]
 
 #   - id: "vizgen_merscope/2022_vizgen_human_colon_cancer_merfish/rep2"
-#     gcloud_bucket: "vz-ffpe-showcase"
-#     dataset_bucket_name: "HumanColonCancerPatient2"
+#     input: "gs://vz-ffpe-showcase/HumanColonCancerPatient2"
 #     dataset_name: "2022 Vizgen Human Colon Cancer MERFISH Patient2"
 #     dataset_url: "https://info.vizgen.com/ffpe-showcase"
 #     dataset_summary: "Human Colon Cancer data from the MERSCOPE FFPE Human Immuno-Oncology Data Release."
@@ -104,8 +95,7 @@ tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \
 #     segmentation_id: ["cell"]
 
 #   - id: "vizgen_merscope/2022_vizgen_human_melanoma_merfish/rep1"
-#     gcloud_bucket: "vz-ffpe-showcase"
-#     dataset_bucket_name: "HumanMelanomaPatient1"
+#     input: "gs://vz-ffpe-showcase/HumanMelanomaPatient1"
 #     dataset_name: "2022 Vizgen Human Melanoma MERFISH Patient1"
 #     dataset_url: "https://info.vizgen.com/ffpe-showcase"
 #     dataset_summary: "Human Melanoma data from the MERSCOPE FFPE Human Immuno-Oncology Data Release."
@@ -114,8 +104,7 @@ tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \
 #     segmentation_id: ["cell"]
 
 #   - id: "vizgen_merscope/2022_vizgen_human_melanoma_merfish/rep2"
-#     gcloud_bucket: "vz-ffpe-showcase"
-#     dataset_bucket_name: "HumanMelanomaPatient2"
+#     input: "gs://vz-ffpe-showcase/HumanMelanomaPatient2"
 #     dataset_name: "2022 Vizgen Human Melanoma MERFISH Patient2"
 #     dataset_url: "https://info.vizgen.com/ffpe-showcase"
 #     dataset_summary: "Human Melanoma data from the MERSCOPE FFPE Human Immuno-Oncology Data Release."
@@ -124,8 +113,7 @@ tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \
 #     segmentation_id: ["cell"]
 
 #   - id: "vizgen_merscope/2022_vizgen_human_ovarian_cancer_merfish/rep1"
-#     gcloud_bucket: "vz-ffpe-showcase"
-#     dataset_bucket_name: "HumanOvarianCancerPatient1"
+#     input: "gs://vz-ffpe-showcase/HumanOvarianCancerPatient1"
 #     dataset_name: "2022 Vizgen Human Ovarian Cancer MERFISH Patient1"
 #     dataset_url: "https://info.vizgen.com/ffpe-showcase"
 #     dataset_summary: "Human Ovarian Cancer data from the MERSCOPE FFPE Human Immuno-Oncology Data Release."
@@ -135,8 +123,7 @@ tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \
 
 #   # Patient 2 has multiple slices
 #   - id: "vizgen_merscope/2022_vizgen_human_ovarian_cancer_merfish/rep2_slice1"
-#     gcloud_bucket: "vz-ffpe-showcase"
-#     dataset_bucket_name: "HumanOvarianCancerPatient2Slice1"
+#     input: "gs://vz-ffpe-showcase/HumanOvarianCancerPatient2Slice1"
 #     dataset_name: "2022 Vizgen Human Ovarian Cancer MERFISH Patient2 Slice1"
 #     dataset_url: "https://info.vizgen.com/ffpe-showcase"
 #     dataset_summary: "Human Ovarian Cancer data (Slice 1) from the MERSCOPE FFPE Human Immuno-Oncology Data Release."
@@ -145,8 +132,7 @@ tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \
 #     segmentation_id: ["cell"]
 
 #   - id: "vizgen_merscope/2022_vizgen_human_ovarian_cancer_merfish/rep2_slice2"
-#     gcloud_bucket: "vz-ffpe-showcase"
-#     dataset_bucket_name: "HumanOvarianCancerPatient2Slice2"
+#     input: "gs://vz-ffpe-showcase/HumanOvarianCancerPatient2Slice2"
 #     dataset_name: "2022 Vizgen Human Ovarian Cancer MERFISH Patient2 Slice2"
 #     dataset_url: "https://info.vizgen.com/ffpe-showcase"
 #     dataset_summary: "Human Ovarian Cancer data (Slice 2) from the MERSCOPE FFPE Human Immuno-Oncology Data Release."
@@ -155,8 +141,7 @@ tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \
 #     segmentation_id: ["cell"]
 
 #   - id: "vizgen_merscope/2022_vizgen_human_ovarian_cancer_merfish/rep2_slice3"
-#     gcloud_bucket: "vz-ffpe-showcase"
-#     dataset_bucket_name: "HumanOvarianCancerPatient2Slice3"
+#     input: "gs://vz-ffpe-showcase/HumanOvarianCancerPatient2Slice3"
 #     dataset_name: "2022 Vizgen Human Ovarian Cancer MERFISH Patient2 Slice3"
 #     dataset_url: "https://info.vizgen.com/ffpe-showcase"
 #     dataset_summary: "Human Ovarian Cancer data (Slice 3) from the MERSCOPE FFPE Human Immuno-Oncology Data Release."
@@ -165,8 +150,7 @@ tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \
 #     segmentation_id: ["cell"]
 
 #   - id: "vizgen_merscope/2022_vizgen_human_prostate_cancer_merfish/rep1"
-#     gcloud_bucket: "vz-ffpe-showcase"
-#     dataset_bucket_name: "HumanProstateCancerPatient1"
+#     input: "gs://vz-ffpe-showcase/HumanProstateCancerPatient1"
 #     dataset_name: "2022 Vizgen Human Prostate Cancer MERFISH Patient1"
 #     dataset_url: "https://info.vizgen.com/ffpe-showcase"
 #     dataset_summary: "Human Prostate Cancer data from the MERSCOPE FFPE Human Immuno-Oncology Data Release."
@@ -175,8 +159,7 @@ tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \
 #     segmentation_id: ["cell"]
 
 #   - id: "vizgen_merscope/2022_vizgen_human_prostate_cancer_merfish/rep2"
-#     gcloud_bucket: "vz-ffpe-showcase"
-#     dataset_bucket_name: "HumanProstateCancerPatient2"
+#     input: "gs://vz-ffpe-showcase/HumanProstateCancerPatient2"
 #     dataset_name: "2022 Vizgen Human Prostate Cancer MERFISH Patient2"
 #     dataset_url: "https://info.vizgen.com/ffpe-showcase"
 #     dataset_summary: "Human Prostate Cancer data from the MERSCOPE FFPE Human Immuno-Oncology Data Release."
@@ -185,8 +168,7 @@ tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \
 #     segmentation_id: ["cell"]
 
 #   - id: "vizgen_merscope/2022_vizgen_human_uterine_cancer_merfish/rep1"
-#     gcloud_bucket: "vz-ffpe-showcase"
-#     dataset_bucket_name: "HumanUterineCancerPatient1"
+#     input: "gs://vz-ffpe-showcase/HumanUterineCancerPatient1"
 #     dataset_name: "2022 Vizgen Human Uterine Cancer MERFISH Patient1"
 #     dataset_url: "https://info.vizgen.com/ffpe-showcase"
 #     dataset_summary: "Human Uterine Cancer data from the MERSCOPE FFPE Human Immuno-Oncology Data Release."
@@ -195,8 +177,7 @@ tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \
 #     segmentation_id: ["cell"]
 
 #   - id: "vizgen_merscope/2022_vizgen_human_uterine_cancer_merfish/rep2"
-#     gcloud_bucket: "vz-ffpe-showcase"
-#     dataset_bucket_name: "HumanUterineCancerPatient2"
+#     input: "gs://vz-ffpe-showcase/HumanUterineCancerPatient2"
 #     dataset_name: "2022 Vizgen Human Uterine Cancer MERFISH Patient2"
 #     dataset_url: "https://info.vizgen.com/ffpe-showcase"
 #     dataset_summary: "Human Uterine Cancer data from the MERSCOPE FFPE Human Immuno-Oncology Data Release."

scripts/create_resources/process_vizgen_merscope.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# get the root of the directory
+REPO_ROOT=$(git rev-parse --show-toplevel)
+
+# ensure that the command below is run from the root of the repository
+cd "$REPO_ROOT"
+
+set -e
+
+publish_dir="s3://openproblems-data/resources/datasets"
+
+# Note that the current download script and processing workflow have a specific default parameter set for the given dataset.
+# No additional datasets are supported by that component/workflow. Therefore the default parameters are used and don't need 
+# to be specified here.
+
+cat > /tmp/params.yaml << HERE
+param_list:
+  - id: wu_human_breast_cancer_sc/2021Wu_human_breast_cancer_sc
+    cancer_subtypes:
+      - HER2+
+      - TNBC
+      - ER+
+
+keep_files: false 
+
+output_dataset: "\$id/dataset.h5ad"
+output_meta: "\$id/dataset_meta.yaml"
+output_state: "\$id/state.yaml"
+publish_dir: "$publish_dir"
+HERE
+
+tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \
+  --revision build/main \
+  --pull-latest \
+  --main-script target/nextflow/datasets/workflows/process_wu_human_breast_cancer_sc/main.nf \
+  --workspace 53907369739130 \
+  --compute-env 6TeIFgV5OY4pJCk8I0bfOh \
+  --params-file /tmp/params.yaml \
+  --config common/nextflow_helpers/labels_tw.config \
+  --labels datasets,wu_human_breast_cancer_sc
+
+aws s3 sync \
+  s3://openproblems-data/resources/datasets/wu_human_breast_cancer_sc/2021Wu_human_breast_cancer_sc \
+  resources/datasets/wu_human_breast_cancer_sc/2021Wu_human_breast_cancer_sc

scripts/create_resources/process_wu_human_breast_cancer_sc.sh

@@ -75,17 +75,18 @@ viash run src/metrics/similarity/config.vsh.yaml -- \
 
 # create a state file
 cat >> $OUT_DIR/state.yaml <<EOL
-output_sp: $OUT_DIR/raw_ist.zarr
-output_sc: $OUT_DIR/scrnaseq_reference.h5ad
-output_segmentation: $OUT_DIR/segmentation.zarr
-output_transcript_assignments: $OUT_DIR/transcript_assignments.zarr
-output_spatial_aggregated_counts: $OUT_DIR/spatial_aggregated_counts.h5ad
-output_cell_volumes: $OUT_DIR/cell_volumes.h5ad
-output_spatial_normalized_counts: $OUT_DIR/spatial_normalized_counts.h5ad
-output_spatial_with_cell_types: $OUT_DIR/spatial_with_cell_types.h5ad
-output_spatial_corrected_counts: $OUT_DIR/spatial_corrected_counts.h5ad
-output_spatial_qc_col: $OUT_DIR/spatial_qc_col.h5ad
-output_score: $OUT_DIR/score.h5ad
+id: mouse_brain_combined
+output_sp: !file raw_ist.zarr
+output_sc: !file scrnaseq_reference.h5ad
+output_segmentation: !file segmentation.zarr
+output_transcript_assignments: !file transcript_assignments.zarr
+output_spatial_aggregated_counts: !file spatial_aggregated_counts.h5ad
+output_cell_volumes: !file cell_volumes.h5ad
+output_spatial_normalized_counts: !file spatial_normalized_counts.h5ad
+output_spatial_with_cell_types: !file spatial_with_cell_types.h5ad
+output_spatial_corrected_counts: !file spatial_corrected_counts.h5ad
+output_spatial_qc_col: !file spatial_qc_col.h5ad
+output_score: !file score.h5ad
 EOL
 
 # sync test resources