Skip to content

Commit 31a37a6

Browse files
LouisK92LouisK92
authored
Merge pull request #38 from openproblems-bio/travaglini_lung_sc
Add travaglini healthy human lung sc data
2 parents 40d5934 + 46afb39 commit 31a37a6

File tree

7 files changed

+449
-0
lines changed

7 files changed

+449
-0
lines changed

‎scripts/create_resources/sc/process_travaglini_human_lung_sc.sh‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#!/bin/bash
2+
3+
# get the root of the directory
4+
REPO_ROOT=$(git rev-parse --show-toplevel)
5+
6+
# ensure that the command below is run from the root of the repository
7+
cd "$REPO_ROOT"
8+
9+
set -e
10+
11+
publish_dir="s3://openproblems-data/resources/datasets"
12+
13+
# Note that the current download script and processing workflow have a specific default parameter set for the given dataset.
14+
# No additional datasets are supported by that component/workflow. Therefore the default parameters are used and don't need
15+
# to be specified here.
16+
17+
cat > /tmp/params.yaml << HERE
18+
param_list:
19+
- id: scrnaseq_for_ist/2020Travaglini_human_lung_sc
20+
21+
keep_files: false
22+
23+
output_dataset: "\$id/dataset.h5ad"
24+
output_meta: "\$id/dataset_meta.yaml"
25+
output_state: "\$id/state.yaml"
26+
publish_dir: "$publish_dir"
27+
HERE
28+
29+
tw launch https://github.com/openproblems-bio/task_ist_preprocessing.git \
30+
--revision build/main \
31+
--pull-latest \
32+
--main-script target/nextflow/datasets/workflows/process_travaglini_human_lung_sc/main.nf \
33+
--workspace 53907369739130 \
34+
--params-file /tmp/params.yaml \
35+
--config common/nextflow_helpers/labels_tw.config \
36+
--labels datasets,travaglini_human_lung_sc
37+
38+
#aws s3 sync \
39+
# s3://openproblems-data/resources/datasets/wu_human_breast_cancer_sc/2021Wu_human_breast_cancer_sc \
40+
# resources/datasets/wu_human_breast_cancer_sc/2021Wu_human_breast_cancer_sc

‎src/datasets/loaders/travaglini_human_lung_sc/config.vsh.yaml‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
name: travaglini_human_lung_sc
2+
namespace: datasets/loaders
3+
4+
argument_groups:
5+
- name: Caching settings
6+
arguments:
7+
- type: boolean
8+
name: --keep_files
9+
required: false
10+
description: Whether to remove the downloaded files after processing.
11+
default: false
12+
- name: Metadata
13+
arguments:
14+
- type: string
15+
name: --dataset_id
16+
description: "A unique identifier for the dataset"
17+
required: false
18+
default: "2020Travaglini_human_lung_sc"
19+
- name: --dataset_name
20+
type: string
21+
description: Nicely formatted name.
22+
required: false
23+
default: "2020Travaglini_human_lung_sc"
24+
- type: string
25+
name: --dataset_url
26+
description: Link to the original source of the dataset.
27+
required: false
28+
default: "https://cellxgene.cziscience.com/collections/6f6d381a-7701-4781-935c-db10d30de293"
29+
- name: --dataset_reference
30+
type: string
31+
description: Bibtex reference of the paper in which the dataset was published.
32+
required: false
33+
default: "https://doi.org/10.1038/s41586-020-2922-4"
34+
- name: --dataset_summary
35+
type: string
36+
description: Short description of the dataset.
37+
required: false
38+
default: "This dataset contains scRNA-seq data from human lung cells."
39+
- name: --dataset_description
40+
type: string
41+
description: Long description of the dataset.
42+
required: false
43+
default: "This dataset contains scRNA-seq data from human lung cells."
44+
- name: --dataset_organism
45+
type: string
46+
description: The organism of the sample in the dataset.
47+
required: false
48+
default: "Homo sapiens"
49+
- name: Outputs
50+
arguments:
51+
- name: "--output"
52+
__merge__: /src/api/file_common_scrnaseq.yaml
53+
direction: output
54+
required: true
55+
56+
resources:
57+
- type: python_script
58+
path: script.py
59+
60+
engines:
61+
- type: docker
62+
image: openproblems/base_python:1
63+
__merge__:
64+
- /src/base/setup_txsim_partial.yaml
65+
- type: native
66+
67+
runners:
68+
- type: executable
69+
- type: nextflow
70+
directives:
71+
label: [highmem, midcpu, midtime]

‎src/datasets/loaders/travaglini_human_lung_sc/script.py‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
from pathlib import Path
2+
import pandas as pd
3+
import anndata as ad
4+
import scanpy as sc
5+
import urllib.request
6+
7+
## VIASH START
8+
par = {
9+
"keep_files": True, # wether to delete the intermediate files
10+
"output": "./temp/datasets/2020Travaglini_human_lung_sc.h5ad",
11+
"dataset_id": "2020Travaglini_human_lung_sc",
12+
"dataset_name": "2020Travaglini_human_lung_sc",
13+
"dataset_url": "https://cellxgene.cziscience.com/collections/6f6d381a-7701-4781-935c-db10d30de293",
14+
"dataset_reference": "https://doi.org/10.1038/s41586-020-2922-4", #TODO: bibtex not doi, also adjust config.vsh.yaml
15+
"dataset_summary": "This dataset contains scRNA-seq data from human lung cells.",
16+
"dataset_description": "This dataset contains scRNA-seq data from human lung cells.",
17+
"dataset_organism": "Homo sapiens"
18+
}
19+
meta = {
20+
"temp_dir": "./temp/datasets/2020Travaglini_human_lung_sc",
21+
}
22+
## VIASH END
23+
24+
# helper variables
25+
TMP_DIR = Path(meta["temp_dir"] or "/tmp")
26+
TMP_DIR.mkdir(parents=True, exist_ok=True)
27+
FILE_URL = "https://datasets.cellxgene.cziscience.com/060e8716-9f0e-4773-9417-582ddc9ba7ab.h5ad"
28+
FILE_PATH = TMP_DIR / "060e8716-9f0e-4773-9417-582ddc9ba7ab.h5ad"
29+
30+
31+
# Download the data
32+
print("Downloading data (~5.5GB)", flush=True)
33+
urllib.request.urlretrieve(FILE_URL, FILE_PATH)
34+
35+
adata = ad.read_h5ad(FILE_PATH)
36+
37+
# Subset to Travaglini/Krasnow dataset
38+
adata = adata[adata.obs['dataset'] == "Krasnow_2020"]
39+
40+
# Filter out cell types with less than 30 cells
41+
ct_value_count = adata.obs['ann_finest_level'].value_counts()
42+
cts_to_keep = ct_value_count[ct_value_count > 30].index.tolist()
43+
adata = adata[adata.obs['ann_finest_level'].isin(cts_to_keep)]
44+
45+
46+
adata.layers["counts"] = adata.raw.X
47+
sc.pp.filter_genes(adata, min_counts=1)
48+
del adata.X
49+
del adata.raw
50+
51+
# Remove cell_type obs column as we'll assign ann_finest_level as cell_type
52+
del adata.obs["cell_type"]
53+
54+
# Rename fields
55+
rename_obs_keys = {
56+
"cell_type": "ann_finest_level",
57+
"batch": "sample",
58+
"assay": "assay",
59+
"assay_ontology_term_id": "assay_ontology_term_id",
60+
"cell_type_ontology_term_id": "cell_type_ontology_term_id",
61+
"development_stage": "development_stage",
62+
"development_stage_ontology_term_id": "development_stage_ontology_term_id",
63+
"disease": "disease",
64+
"disease_ontology_term_id": "disease_ontology_term_id",
65+
"is_primary_data": "is_primary_data",
66+
"self_reported_ethnicity": "self_reported_ethnicity",
67+
"self_reported_ethnicity_ontology_term_id": "self_reported_ethnicity_ontology_term_id",
68+
"sex": "sex",
69+
"sex_ontology_term_id": "sex_ontology_term_id",
70+
"suspension_type": "suspension_type",
71+
"tissue": "tissue",
72+
"tissue_ontology_term_id": "tissue_ontology_term_id",
73+
}
74+
adata.obs = adata.obs.rename(columns={old:new for new,old in rename_obs_keys.items()})
75+
76+
# Add additional information to obs
77+
#TODO: Finish up the terms according to the ontology
78+
#Ontology schema currently (13.03.2025) used in openproblems (CELLxGENE schema v4.0.0):
79+
#https://github.com/chanzuckerberg/single-cell-curation/blob/main/schema/4.0.0/schema.md
80+
#(mentioned here: https://openproblems.bio/documentation/reference/openproblems/src-datasets#file-format:-raw-dataset)
81+
store_info = {
82+
"dataset_id": "2020Travaglini_human_lung_sc",
83+
"organism": "Homo sapiens",
84+
# "organism_ontology_term_id": "NCBITaxon:10090", #TODO: ontology
85+
}
86+
for key, value in store_info.items():
87+
adata.obs[key] = pd.Categorical([value] * adata.n_obs, categories=[value])
88+
89+
# Remove undesired columns
90+
for key in adata.obs.columns:
91+
if (key not in rename_obs_keys.keys()) and (key not in store_info.keys()):
92+
print(f"Removing .obs['{key}']")
93+
del adata.obs[key]
94+
95+
# Var
96+
adata.var["gene_symbol"] = adata.var["feature_name"]
97+
adata.var["feature_id"] = adata.var_names
98+
adata.var_names = adata.var["feature_name"]
99+
adata.var_names = adata.var_names.astype(str)
100+
adata.var_names_make_unique()
101+
adata.var.index.name = None
102+
103+
# Uns
104+
for key in ["dataset_id", "dataset_name", "dataset_url", "dataset_reference", "dataset_summary", "dataset_description", "dataset_organism"]:
105+
adata.uns[key] = par[key]
106+
107+
# Delete files if requested
108+
if not par["keep_files"]:
109+
print("Removing files", flush=True)
110+
if FILE_PATH.exists():
111+
print("\t...", FILE_PATH, flush=True)
112+
FILE_PATH.unlink()
113+
114+
print("Writing adata", flush=True)
115+
adata.write_h5ad(par["output"], compression="gzip")

‎src/datasets/workflows/travaglini_human_lung_sc/config.vsh.yaml‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
name: process_travaglini_human_lung_sc
2+
namespace: datasets/workflows
3+
4+
argument_groups:
5+
- name: Caching settings
6+
arguments:
7+
- type: boolean
8+
name: --keep_files
9+
required: false
10+
description: Whether to remove the downloaded files after processing.
11+
default: false
12+
- name: Metadata
13+
arguments:
14+
- type: string
15+
name: --dataset_id
16+
description: "A unique identifier for the dataset"
17+
required: false
18+
default: "2020Travaglini_human_lung_sc"
19+
- name: --dataset_name
20+
type: string
21+
description: Nicely formatted name.
22+
required: false
23+
default: "2020Travaglini_human_lung_sc"
24+
- type: string
25+
name: --dataset_url
26+
description: Link to the original source of the dataset.
27+
required: false
28+
default: "https://cellxgene.cziscience.com/collections/6f6d381a-7701-4781-935c-db10d30de293"
29+
- name: --dataset_reference
30+
type: string
31+
description: Bibtex reference of the paper in which the dataset was published.
32+
required: false
33+
default: "https://doi.org/10.1038/s41586-020-2922-4"
34+
- name: --dataset_summary
35+
type: string
36+
description: Short description of the dataset.
37+
required: false
38+
default: "This dataset contains scRNA-seq data from human lung cells."
39+
- name: --dataset_description
40+
type: string
41+
description: Long description of the dataset.
42+
required: false
43+
default: "This dataset contains scRNA-seq data from human lung cells."
44+
- name: --dataset_organism
45+
type: string
46+
description: The organism of the sample in the dataset.
47+
required: false
48+
default: "Homo sapiens"
49+
- name: Outputs
50+
arguments:
51+
- name: "--output_dataset"
52+
__merge__: /src/api/file_common_scrnaseq.yaml
53+
direction: output
54+
required: true
55+
default: "$id/dataset.h5ad"
56+
- name: "--output_meta"
57+
direction: "output"
58+
type: file
59+
description: "Dataset metadata"
60+
default: "$id/dataset_meta.yaml"
61+
62+
resources:
63+
- type: nextflow_script
64+
path: main.nf
65+
entrypoint: run_wf
66+
- path: /common/nextflow_helpers/helper.nf
67+
68+
dependencies:
69+
- name: datasets/loaders/travaglini_human_lung_sc
70+
- name: datasets/normalization/log_cp
71+
repository: openproblems
72+
- name: datasets/processors/pca
73+
repository: openproblems
74+
- name: datasets/processors/hvg
75+
repository: openproblems
76+
- name: datasets/processors/knn
77+
repository: openproblems
78+
- name: utils/extract_uns_metadata
79+
repository: openproblems
80+
81+
runners:
82+
- type: nextflow
83+
directives:
84+
label: [midcpu, midmem, hightime]

0 commit comments

Comments
 (0)