Skip to content

Generating output #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Sigfried merged 1 commit into from
Jul 14, 2025
Merged

Generating output #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
0 auto_gen_concept_templates → auto_gen_concept_templates/README.md
View file
Open in desktop
File renamed without changes.
106 changes: 106 additions & 0 deletions auto_gen_concept_templates/autogen.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
import gspread
from gspread_dataframe import set_with_dataframe, get_as_dataframe
import pandas as pd

mapping_sources = [
{
'spreadsheet_name': 'AIREADI MOCA Data Dictionary and Mappings v0.3',
'worksheet_name': 'MOCA Data Dictionary with Extensions',
'process': True,
'tag': 'MOCA',
},
{
'spreadsheet_name': 'REDCap Data Dictionary and OMOP Mappings',
'worksheet_name': '_master REDCap Data Dictionary with Extensions',
'process': False,
'tag': 'RedCap',
},
]
outputs = [
{
'name': 'concept_manual.csv',
'columns': {
'concept_name': {'MOCA': 'TARGET_CONCEPT_NAME'},
'SRC_CODE': {'default': ''},
'concept_id': {'MOCA': 'TARGET_CONCEPT_ID'},
'vocabulary_id': {'MOCA': 'TARGET_VOCABULARY_ID'},
'domain_id': {'MOCA': 'TARGET_DOMAIN_ID'},
'v6_domain_id': {'default': 'survey_conduct'},
'concept_class_id': {'MOCA': 'TARGET_CONCEPT_CLASS_ID'},
'standard_concept': {'MOCA': 'TARGET_STANDARD_CONCEPT'},
'valid_start_date': {'default': '1/1/1970'},
'valid_end_date': {'default': ''},
'invalid_reason': {'default': ''},
}
},
{
'name': 'concept_relationship_manual.csv',
'columns': {
'concept_name': {'MOCA': 'TARGET_CONCEPT_NAME'},
'concept_id_1': {'MOCA': 'TARGET_CONCEPT_ID'},
'SRC_CODE': {'default': ''},
'vocabulary_id_1': {'MOCA': 'TARGET_VOCABULARY_ID'},
'relationship_id': {'default': ''},
'concept_id_2': {'default': '606671'},
'temp name': {'default': 'Montreal Cognitive Assessment version 8.1'},
'temp domain': {'default': 'Measurement'},
'vocabulary_id_2': {'default': 'SNOMED'},
'temp class': {'default': 'Staging / Scales'},
'concept_code_2': {'default': '1148423001'},
'temp standard': {'default': 'S'},
'relationship_valid_start_date': {'default': ''},
'relationship_valid_end_date': {'default': ''},
'invalid_reason': {'default': ''},
'confidence': {'default': '1'},
'predicate_id': {'default': 'skos:broadMatch'},
'mapping_source': {'default': 'ManualMapping'},
'mapping_justification': {'default': 'ManualMappingCuration'},
'mapping_tool': {'default': ''},
'Notes': {'default': ''},
}
},
]


def main():
gc = gspread.service_account()

for source in mapping_sources:
if not source['process']:
continue

spreadsheet = gc.open(source['spreadsheet_name'])
mapping = spreadsheet.worksheet(source['worksheet_name'])
df_all = get_as_dataframe(mapping)
df = df_all.copy()[df_all['TARGET_CONCEPT_ID'] > 2000000000]
tag = source['tag']

for ofile in outputs:
columns = ofile['columns']
result_data = {}

# Process each column in the output specification
for col_name, source_def in columns.items():
if tag in source_def:
# Use the source column from the dataframe
source_col = source_def[tag]
if source_col in df.columns:
result_data[col_name] = df[source_col].values
else:
print(f"Warning: Column '{source_col}' not found in source data")
result_data[col_name] = [source_def.get('default', '')] * len(df)
else:
# Use default value for all rows
default_value = source_def.get('default', '')
result_data[col_name] = [default_value] * len(df)

# Create DataFrame from the result data
result_df = pd.DataFrame(result_data)

# Save to CSV
filename = 'output/' + ofile['name']
result_df.to_csv(filename, index=False)
print(f"Saved {filename} with {len(result_df)} rows")

if __name__ == "__main__":
main()
31 changes: 31 additions & 0 deletions auto_gen_concept_templates/output/concept_manual.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
concept_name,SRC_CODE,concept_id,vocabulary_id,domain_id,v6_domain_id,concept_class_id,standard_concept,valid_start_date,valid_end_date,invalid_reason
Trail drawings score,,2005200344.0,SNOMED,Measurement,survey_conduct,Staging / Scales,S,1/1/1970,,
Time taken to do trails in seconds,,2005200345.0,AIREADI-MOCA,Measurement,survey_conduct,Assessment Item ,S,1/1/1970,,
Cube drawings score,,2005200346.0,AIREADI-MOCA,Measurement,survey_conduct,Assessment Item ,S,1/1/1970,,
Time taken to do cube in seconds,,2005200347.0,AIREADI-MOCA,Measurement,survey_conduct,Assessment Item ,S,1/1/1970,,
Clock drawings score,,2005200348.0,LOINC,Observation,survey_conduct,Survey,S,1/1/1970,,
Time taken to do clock in seconds,,2005200349.0,AIREADI-MOCA,Measurement,survey_conduct,Assessment Item ,S,1/1/1970,,
Animal name recall task score,,2005200350.0,AIREADI-MOCA,Measurement,survey_conduct,Observable Entity,S,1/1/1970,,
Time in seconds to perform the animal name recall task,,2005200351.0,AIREADI-MOCA,Measurement,survey_conduct,Assessment Item ,S,1/1/1970,,
Words correctly repeated (1st trial) score,,2005200352.0,AIREADI-MOCA,Measurement,survey_conduct,Assessment Item ,S,1/1/1970,,
Time in seconds to perform the words correctly repeated (1st trial),,2005200353.0,AIREADI-MOCA,Measurement,survey_conduct,Assessment Item ,S,1/1/1970,,
Words correctly repeated (2nd trial) score,,2005200354.0,AIREADI-MOCA,Measurement,survey_conduct,Assessment Item ,S,1/1/1970,,
Time in seconds to perform the words correctly repeated (2nd trial),,2005200355.0,AIREADI-MOCA,Measurement,survey_conduct,Assessment Item ,S,1/1/1970,,
Repeating digits score,,2005200356.0,SNOMED,Observation,survey_conduct,Observable Entity,S,1/1/1970,,
Time in seconds to perform the repeating digits task,,2005200357.0,AIREADI-MOCA,Measurement,survey_conduct,Assessment Item ,S,1/1/1970,,
Letter tapping score,,2005200358.0,AIREADI-MOCA,Measurement,survey_conduct,Assessment Item ,S,1/1/1970,,
Time in seconds to perform the letter tapping task,,2005200359.0,AIREADI-MOCA,Measurement,survey_conduct,Assessment Item ,S,1/1/1970,,
Serial 7 subtraction score,,2005200360.0,AIREADI-MOCA,Measurement,survey_conduct,Assessment Item ,S,1/1/1970,,
Time in seconds to perform the serial 7 subtraction task,,2005200361.0,AIREADI-MOCA,Measurement,survey_conduct,Assessment Item ,S,1/1/1970,,
Repeats sentences II score,,2005200362.0,AIREADI-MOCA,Measurement,survey_conduct,Assessment Item ,S,1/1/1970,,
Time in seconds to perform repeats sentences II task,,2005200363.0,AIREADI-MOCA,Measurement,survey_conduct,Assessment Item ,S,1/1/1970,,
Maximal one-minute recall of words beginning with 'F' score,,2005200364.0,AIREADI-MOCA,Measurement,survey_conduct,Assessment Item ,S,1/1/1970,,
Time in seconds to perform the words beginning with 'F' recall task,,2005200365.0,AIREADI-MOCA,Measurement,survey_conduct,Assessment Item ,S,1/1/1970,,
MOCA abstraction score,,2005200366.0,AIREADI-MOCA,Measurement,survey_conduct,Assessment Item ,S,1/1/1970,,
Time in seconds to perform MOCA abstraction task,,2005200367.0,AIREADI-MOCA,Measurement,survey_conduct,Assessment Item ,S,1/1/1970,,
MOCA delayed recall with no cue score,,2005200368.0,AIREADI-MOCA,Measurement,survey_conduct,Assessment Item ,S,1/1/1970,,
Time in seconds to perform the MOCA delayed recall with no cue task,,2005200369.0,AIREADI-MOCA,Measurement,survey_conduct,Assessment Item ,S,1/1/1970,,
"MOCA orientation date, month, year, day, place, city score",,2005200370.0,AIREADI-MOCA,Measurement,survey_conduct,Assessment Item ,S,1/1/1970,,
"Time in seconds to perform the MOCA orientation date, month, year, day, place, city task",,2005200371.0,AIREADI-MOCA,Measurement,survey_conduct,Assessment Item ,S,1/1/1970,,
Age in years (3-4 year span),,2005200372.0,AIREADI-MOCA,Observation,survey_conduct,Observable Entity,S,1/1/1970,,
"MOCA delayed recall with no cue, cue, and multiple choice cue (Combined score)",,2005200373.0,AIREADI-MOCA,Measurement,survey_conduct,Assessment Item ,S,1/1/1970,,
31 changes: 31 additions & 0 deletions auto_gen_concept_templates/output/concept_relationship_manual.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
concept_name,concept_id_1,SRC_CODE,vocabulary_id_1,relationship_id,concept_id_2,temp name,temp domain,vocabulary_id_2,temp class,concept_code_2,temp standard,relationship_valid_start_date,relationship_valid_end_date,invalid_reason,confidence,predicate_id,mapping_source,mapping_justification,mapping_tool,Notes
Trail drawings score,2005200344.0,,SNOMED,,606671,Montreal Cognitive Assessment version 8.1,Measurement,SNOMED,Staging / Scales,1148423001,S,,,,1,skos:broadMatch,ManualMapping,ManualMappingCuration,,
Time taken to do trails in seconds,2005200345.0,,AIREADI-MOCA,,606671,Montreal Cognitive Assessment version 8.1,Measurement,SNOMED,Staging / Scales,1148423001,S,,,,1,skos:broadMatch,ManualMapping,ManualMappingCuration,,
Cube drawings score,2005200346.0,,AIREADI-MOCA,,606671,Montreal Cognitive Assessment version 8.1,Measurement,SNOMED,Staging / Scales,1148423001,S,,,,1,skos:broadMatch,ManualMapping,ManualMappingCuration,,
Time taken to do cube in seconds,2005200347.0,,AIREADI-MOCA,,606671,Montreal Cognitive Assessment version 8.1,Measurement,SNOMED,Staging / Scales,1148423001,S,,,,1,skos:broadMatch,ManualMapping,ManualMappingCuration,,
Clock drawings score,2005200348.0,,LOINC,,606671,Montreal Cognitive Assessment version 8.1,Measurement,SNOMED,Staging / Scales,1148423001,S,,,,1,skos:broadMatch,ManualMapping,ManualMappingCuration,,
Time taken to do clock in seconds,2005200349.0,,AIREADI-MOCA,,606671,Montreal Cognitive Assessment version 8.1,Measurement,SNOMED,Staging / Scales,1148423001,S,,,,1,skos:broadMatch,ManualMapping,ManualMappingCuration,,
Animal name recall task score,2005200350.0,,AIREADI-MOCA,,606671,Montreal Cognitive Assessment version 8.1,Measurement,SNOMED,Staging / Scales,1148423001,S,,,,1,skos:broadMatch,ManualMapping,ManualMappingCuration,,
Time in seconds to perform the animal name recall task,2005200351.0,,AIREADI-MOCA,,606671,Montreal Cognitive Assessment version 8.1,Measurement,SNOMED,Staging / Scales,1148423001,S,,,,1,skos:broadMatch,ManualMapping,ManualMappingCuration,,
Words correctly repeated (1st trial) score,2005200352.0,,AIREADI-MOCA,,606671,Montreal Cognitive Assessment version 8.1,Measurement,SNOMED,Staging / Scales,1148423001,S,,,,1,skos:broadMatch,ManualMapping,ManualMappingCuration,,
Time in seconds to perform the words correctly repeated (1st trial),2005200353.0,,AIREADI-MOCA,,606671,Montreal Cognitive Assessment version 8.1,Measurement,SNOMED,Staging / Scales,1148423001,S,,,,1,skos:broadMatch,ManualMapping,ManualMappingCuration,,
Words correctly repeated (2nd trial) score,2005200354.0,,AIREADI-MOCA,,606671,Montreal Cognitive Assessment version 8.1,Measurement,SNOMED,Staging / Scales,1148423001,S,,,,1,skos:broadMatch,ManualMapping,ManualMappingCuration,,
Time in seconds to perform the words correctly repeated (2nd trial),2005200355.0,,AIREADI-MOCA,,606671,Montreal Cognitive Assessment version 8.1,Measurement,SNOMED,Staging / Scales,1148423001,S,,,,1,skos:broadMatch,ManualMapping,ManualMappingCuration,,
Repeating digits score,2005200356.0,,SNOMED,,606671,Montreal Cognitive Assessment version 8.1,Measurement,SNOMED,Staging / Scales,1148423001,S,,,,1,skos:broadMatch,ManualMapping,ManualMappingCuration,,
Time in seconds to perform the repeating digits task,2005200357.0,,AIREADI-MOCA,,606671,Montreal Cognitive Assessment version 8.1,Measurement,SNOMED,Staging / Scales,1148423001,S,,,,1,skos:broadMatch,ManualMapping,ManualMappingCuration,,
Letter tapping score,2005200358.0,,AIREADI-MOCA,,606671,Montreal Cognitive Assessment version 8.1,Measurement,SNOMED,Staging / Scales,1148423001,S,,,,1,skos:broadMatch,ManualMapping,ManualMappingCuration,,
Time in seconds to perform the letter tapping task,2005200359.0,,AIREADI-MOCA,,606671,Montreal Cognitive Assessment version 8.1,Measurement,SNOMED,Staging / Scales,1148423001,S,,,,1,skos:broadMatch,ManualMapping,ManualMappingCuration,,
Serial 7 subtraction score,2005200360.0,,AIREADI-MOCA,,606671,Montreal Cognitive Assessment version 8.1,Measurement,SNOMED,Staging / Scales,1148423001,S,,,,1,skos:broadMatch,ManualMapping,ManualMappingCuration,,
Time in seconds to perform the serial 7 subtraction task,2005200361.0,,AIREADI-MOCA,,606671,Montreal Cognitive Assessment version 8.1,Measurement,SNOMED,Staging / Scales,1148423001,S,,,,1,skos:broadMatch,ManualMapping,ManualMappingCuration,,
Repeats sentences II score,2005200362.0,,AIREADI-MOCA,,606671,Montreal Cognitive Assessment version 8.1,Measurement,SNOMED,Staging / Scales,1148423001,S,,,,1,skos:broadMatch,ManualMapping,ManualMappingCuration,,
Time in seconds to perform repeats sentences II task,2005200363.0,,AIREADI-MOCA,,606671,Montreal Cognitive Assessment version 8.1,Measurement,SNOMED,Staging / Scales,1148423001,S,,,,1,skos:broadMatch,ManualMapping,ManualMappingCuration,,
Maximal one-minute recall of words beginning with 'F' score,2005200364.0,,AIREADI-MOCA,,606671,Montreal Cognitive Assessment version 8.1,Measurement,SNOMED,Staging / Scales,1148423001,S,,,,1,skos:broadMatch,ManualMapping,ManualMappingCuration,,
Time in seconds to perform the words beginning with 'F' recall task,2005200365.0,,AIREADI-MOCA,,606671,Montreal Cognitive Assessment version 8.1,Measurement,SNOMED,Staging / Scales,1148423001,S,,,,1,skos:broadMatch,ManualMapping,ManualMappingCuration,,
MOCA abstraction score,2005200366.0,,AIREADI-MOCA,,606671,Montreal Cognitive Assessment version 8.1,Measurement,SNOMED,Staging / Scales,1148423001,S,,,,1,skos:broadMatch,ManualMapping,ManualMappingCuration,,
Time in seconds to perform MOCA abstraction task,2005200367.0,,AIREADI-MOCA,,606671,Montreal Cognitive Assessment version 8.1,Measurement,SNOMED,Staging / Scales,1148423001,S,,,,1,skos:broadMatch,ManualMapping,ManualMappingCuration,,
MOCA delayed recall with no cue score,2005200368.0,,AIREADI-MOCA,,606671,Montreal Cognitive Assessment version 8.1,Measurement,SNOMED,Staging / Scales,1148423001,S,,,,1,skos:broadMatch,ManualMapping,ManualMappingCuration,,
Time in seconds to perform the MOCA delayed recall with no cue task,2005200369.0,,AIREADI-MOCA,,606671,Montreal Cognitive Assessment version 8.1,Measurement,SNOMED,Staging / Scales,1148423001,S,,,,1,skos:broadMatch,ManualMapping,ManualMappingCuration,,
"MOCA orientation date, month, year, day, place, city score",2005200370.0,,AIREADI-MOCA,,606671,Montreal Cognitive Assessment version 8.1,Measurement,SNOMED,Staging / Scales,1148423001,S,,,,1,skos:broadMatch,ManualMapping,ManualMappingCuration,,
"Time in seconds to perform the MOCA orientation date, month, year, day, place, city task",2005200371.0,,AIREADI-MOCA,,606671,Montreal Cognitive Assessment version 8.1,Measurement,SNOMED,Staging / Scales,1148423001,S,,,,1,skos:broadMatch,ManualMapping,ManualMappingCuration,,
Age in years (3-4 year span),2005200372.0,,AIREADI-MOCA,,606671,Montreal Cognitive Assessment version 8.1,Measurement,SNOMED,Staging / Scales,1148423001,S,,,,1,skos:broadMatch,ManualMapping,ManualMappingCuration,,
"MOCA delayed recall with no cue, cue, and multiple choice cue (Combined score)",2005200373.0,,AIREADI-MOCA,,606671,Montreal Cognitive Assessment version 8.1,Measurement,SNOMED,Staging / Scales,1148423001,S,,,,1,skos:broadMatch,ManualMapping,ManualMappingCuration,,
Loading