conda create --name access_data_analysis python=3
conda activate access_data_analysis
conda install r-essentials r-base r-argparse r-ggpubr r-ggthemes r-plotly r-kableextra r-htmlwidgets r-dt
pip install genotype-variants

> Rscript R/compile_reads.R -m $PATH/TO/master_file.csv -o $PATH/TO/results_folder

> Rscript R/filter_calls.R -m $PATH/TO/master_file.csv -o $PATH/TO/results_folder

> Rscript R/SV_incorporation.R -m $PATH/TO/manifest_file.tsv -o $PATH/TO/results_folder

> Rscript R/CNA_processing.R -m $PATH/TO/manifest_file.tsv -o $PATH/TO/results_folder

> Rscript R/plot_all_events.R -m $PATH/TO/manifest_file.tsv -o $PATH/TO/results_folder

> Rscript ~/github/access_data_analysis/reports/create_report.R -md -t ~/github/access_data_analysis/reports/template_days.Rmd -p C-L6H8E2 -r ../results_20Jan2023/results_stringent_hc/C-L6H8E2_SNV_table.csv -tt "Melanoma" -m ../manifest_noDate_days.tsv -o C-L6H8E2_days.html -rc ../results_20Jan2023/CNA_final_call_set -d P-0022907 -ds P-0022907-T01-IM6 -dm /juno/work/ccs/shared/resources/impact/facets/all/P-00229/P-0022907-T01-IM6_P-0022907-N01-IM6/default/P-0022907-T01-IM6_P-0022907-N01-IM6.ccf.maf

Rscript swimmer_single_treatment.R -i input_data.txt -o output_plot.pdf -t days

Rscript swimmer_multi_treatment.R -m metadata.xlsx -o /path/to/output -c blue,red,green -t weeks

Rscript dates2days.R -i input_data.txt -o output_data.txt

install.packages(c("dplyr", "ggplot2", "lubridate", "argparse", "readr", "readxl", "tidyr", "scales", "gridExtra", "cowplot"))

install.packages(c("ggplot2", "gridExtra", "tidyr", "dplyr", "sqldf", "RSQLite", "readr", "argparse", "plotly", "htmlwidgets", "purrr"))

Rscript vaf_overview_plot.R -o /path/to/output -v /path/to/variants.maf -c /path/to/clinical.tsv -y mean -n 10

Hugo_Symbol,Chromosome,Start_Position,End_Position,Tumor_Sample_Barcode,Variant_Classification,HGVSp_Short,Reference_Allele,Tumor_Seq_Allele2,D_t_alt_count_fragment

Rscript R/SV_incorporation.R -h                                     
usage: R/SV_incorporation.R [-h] [-m MASTERREF] [-o RESULTSDIR] [-dmp DMPDIR]
                            [-c CRITERIA]

optional arguments:
  -h, --help            show this help message and exit
  -m MASTERREF, --masterref MASTERREF
                        File path to master reference file
  -o RESULTSDIR, --resultsdir RESULTSDIR
                        Output directory
  -dmp DMPDIR, --dmpdir DMPDIR
                        Directory of clinical DMP IMPACT repository [default]
  -genes GENELIST, --genelist GENELIST
                        File path to genes covered by ACCESS [default]
  -c CRITERIA, --criteria CRITERIA
                        Calling criteria [default]

Rscript R/CNA_processing.R -h                                       
usage: R/CNA_processing.R [-h] [-m MASTERREF] [-o RESULTSDIR] [-dmp DMPDIR]

optional arguments:
  -h, --help            show this help message and exit
  -m MASTERREF, --masterref MASTERREF
                        File path to master reference file
  -o RESULTSDIR, --resultsdir RESULTSDIR
                        Output directory
  -dmp DMPDIR, --dmpdir DMPDIR
                        Directory of clinical DMP IMPACT repository [default]

python get_cbioportal_variants.py  subset-maf --sid "Test1" --sid "Test2" --sid "Test3"

python get_cbioportal_variants.py  subset-maf --ids /path/to/ids.txt

Usage: get_cbioportal_variants.py [OPTIONS] COMMAND [ARGS]...

Options:
  --install-completion  Install completion for the current shell.
  --show-completion     Show completion for the current shell, to copy it or
                        customize the installation.

  --help                Show this message and exit.

Commands:
  subset-cna  Subset data_CNA.txt file for given set of sample ids.
  subset-cpt  Subset data_clinical_patient.txt file for given set of
              patient...

  subset-cst  Subset data_clinical_samples.txt file for given set of sample...
  subset-maf  Subset MAF/TSV file and mark if an alteration is covered by...
  subset-sv   Subset data_sv.txt file for given set of sample ids.

Usage: get_cbioportal_variants.py subset-cpt [OPTIONS]

  Subset data_clinical_patient.txt file for given set of patient ids.

  Tool to do the following operations: A. Get subset of clinical information
  for samples based on PATIENT_ID in data_clinical_patient.txt file

  Requirement: pandas; typing; typer; bed_lookup(https://github.com/msk-
  access/python_bed_lookup)

Options:
  -p, --cpt FILE    Clinical Patient file generated by cBioportal repo
                    [default: /work/access/production/resources/cbioportal/cur
                    rent/msk_solid_heme/data_clinical_patient.txt]

  -i, --ids PATH    List of ids to search for in the 'PATIENT_ID' column.
                    Header of this file is 'sample_id'  [default: ]

  --sid TEXT        Identifiers to search for in the 'PATIENT_ID' column. Can
                    be given multiple times  [default: ]

  -n, --name TEXT   Name of the output file  [default:
                    output_clinical_patient.txt]

  -c, --cname TEXT  Name of the column header to be used for sub-setting
                    [default: PATIENT_ID]

  --help            Show this message and exit.

Usage: get_cbioportal_variants.py subset-cst [OPTIONS]

  Subset data_clinical_samples.txt file for given set of sample ids.

  Tool to do the following operations: A. Get subset of clinical information
  for samples based on SAMPLE_ID in data_clinical_sample.txt file

  Requirement: pandas; typing; typer; bed_lookup(https://github.com/msk-
  access/python_bed_lookup)

Options:
  -s, --cst FILE    Clinical Sample file generated by cBioportal repo
                    [default: /work/access/production/resources/cbioportal/cur
                    rent/msk_solid_heme/data_clinical_sample.txt]

  -i, --ids PATH    List of ids to search for in the 'SAMPLE_ID' column.
                    Header of this file is 'sample_id'  [default: ]

  --sid TEXT        Identifiers to search for in the 'SAMPLE_ID' column. Can
                    be given multiple times  [default: ]

  -n, --name TEXT   Name of the output file  [default:
                    output_clinical_samples.txt]

  -c, --cname TEXT  Name of the column header to be used for sub-setting
                    [default: SAMPLE_ID]

  --help            Show this message and exit.

Usage: get_cbioportal_variants.py subset-cna [OPTIONS]

  Subset data_CNA.txt file for given set of sample ids.

  Tool to do the following operations: A. Get subset of samples based on
  column header in data_CNA.txt file

  Requirement: pandas; typing; typer; bed_lookup(https://github.com/msk-
  access/python_bed_lookup)

Options:
  -c, --cna FILE   Copy Number Variant file generated by cBioportal repo
                   [default: /work/access/production/resources/cbioportal/curr
                   ent/msk_solid_heme/data_CNA.txt]

  -i, --ids PATH   List of ids to search for in the 'header' of the file.
                   Header of this file is 'sample_id'  [default: ]

  --sid TEXT       Identifiers to search for in the 'header' of the file. Can
                   be given multiple times  [default: ]

  -n, --name TEXT  Name of the output file  [default: output_CNA.txt]
  --help           Show this message and exit.

Usage: get_cbioportal_variants.py subset-sv [OPTIONS]

  Subset data_sv.txt file for given set of sample ids.

  Tool to do the following operations: A. Get subset of structural variants
  based on Sample_ID in data_sv.txt file

  Requirement: pandas; typing; typer; bed_lookup(https://github.com/msk-
  access/python_bed_lookup)

Options:
  -s, --sv FILE     Structural Variant file generated by cBioportal repo
                    [default: /work/access/production/resources/cbioportal/cur
                    rent/msk_solid_heme/data_sv.txt]

  -i, --ids PATH    List of ids to search for in the 'Sample_ID' column.
                    Header of this file is 'sample_id'  [default: ]

  --sid TEXT        Identifiers to search for in the 'Sample_ID' column. Can
                    be given multiple times  [default: ]

  -n, --name TEXT   Name of the output file  [default: output_sv.txt]
  -c, --cname TEXT  Name of the column header to be used for sub-setting
                    [default: Sample_ID]

  --help            Show this message and exit.

Usage: get_cbioportal_variants.py subset-maf [OPTIONS]

  Subset MAF/TSV file and mark if an alteration is covered by BED file or
  not

  Tool to do the following operations: A. Get subset of variants based on
  Tumor_Sample_Barcode in data_mutations_extended.txt file B. Mark the
  variants as overlapping with BED file as covered [yes/no], by appending
  "covered" column to the subset MAF

  Requirement: pandas; typing; typer; bed_lookup(https://github.com/msk-
  access/python_bed_lookup)

Options:
  -m, --maf FILE    MAF file generated by cBioportal repo  [default: /work/acc
                    ess/production/resources/cbioportal/current/msk_solid_heme
                    /data_mutations_extended.txt]

  -i, --ids PATH    List of ids to search for in the 'Tumor_Sample_Barcode'
                    column. Header of this file is 'sample_id'  [default: ]

  --sid TEXT        Identifiers to search for in the 'Tumor_Sample_Barcode'
                    column. Can be given multiple times  [default: ]

  -b, --bed FILE    BED file to find overlapping variants  [default:
                    /work/access/production/resources/msk-
                    access/current/regions_of_interest/current/MSK-
                    ACCESS-v1_0-probe-A.sorted.bed]

  -n, --name TEXT   Name of the output file  [default: output.maf]
  -c, --cname TEXT  Name of the column header to be used for sub-setting
                    [default: Tumor_Sample_Barcode]

  --help            Show this message and exit.

def read_tsv(tsv)

def read_ids(sid, ids)

def filter_by_columns(sid, tsv_df)

def filter_by_rows(sid, tsv_df, col_name)

def read_bed(bed)

def check_if_covered(bedObj, mafObj)

def get_row(tsv_file)

Rscript R/compile_reads.R -h                                        
usage: R/compile_reads.R [-h] [-m MASTERREF] [-o RESULTSDIR]
                         [-pb POOLEDBAMDIR] [-fa FASTAPATH]
                         [-gt GENOTYPERPATH] [-dmp DMPDIR] [-mb MIRRORBAMDIR]
                         [-dmpk DMPKEYPATH]

optional arguments:
  -h, --help            show this help message and exit
  -m MASTERREF, --masterref MASTERREF
                        File path to master reference file
  -o RESULTSDIR, --resultsdir RESULTSDIR
                        Output directory
  -pb POOLEDBAMDIR, --pooledbamdir POOLEDBAMDIR
                        Directory for all pooled bams [default]
  -fa FASTAPATH, --fastapath FASTAPATH
                        Reference fasta path [default]
  -gt GENOTYPERPATH, --genotyperpath GENOTYPERPATH
                        Genotyper executable path [default]
  -dmp DMPDIR, --dmpdir DMPDIR
                        Directory of clinical DMP IMPACT repository [default]
  -mb MIRRORBAMDIR, --mirrorbamdir MIRRORBAMDIR
                        Mirror BAM file directory [default]
  -dmpk DMPKEYPATH, --dmpkeypath DMPKEYPATH
                        DMP mirror BAM key file [default]

Rscript R/compile_reads_all.R -h
usage: R/compile_reads_all.R [-h] [-m MASTERREF] [-o RESULTSDIR]
                             [-pid PROJECTID] [-pb POOLEDBAMDIR]
                             [-fa FASTAPATH] [-gt GENOTYPERPATH] [-dmp DMPDIR]
                             [-mb MIRRORBAMDIR] [-mab MIRRORACCESSBAMDIR]
                             [-dmpk DMPKEYPATH] [-dmpak DMPACCESSKEYPATH]

optional arguments:
  -h, --help            show this help message and exit
  -m MASTERREF, --masterref MASTERREF
                        File path to master reference file
  -o RESULTSDIR, --resultsdir RESULTSDIR
                        Output directory
  -pid PROJECTID, --projectid PROJECTID
                        Project ID for submitted jobs involved in this run
  -pb POOLEDBAMDIR, --pooledbamdir POOLEDBAMDIR
                        Directory for all pooled bams [default]
  -fa FASTAPATH, --fastapath FASTAPATH
                        Reference fasta path [default]
  -gt GENOTYPERPATH, --genotyperpath GENOTYPERPATH
                        Genotyper executable path [default]
  -dmp DMPDIR, --dmpdir DMPDIR
                        Directory of clinical DMP repository [default]
  -mb MIRRORBAMDIR, --mirrorbamdir MIRRORBAMDIR
                        Mirror BAM file directory [default]
  -mab MIRRORACCESSBAMDIR, --mirroraccessbamdir MIRRORACCESSBAMDIR
                        Mirror BAM file directory for MSK-ACCESS [default]
  -dmpk DMPKEYPATH, --dmpkeypath DMPKEYPATH
                        DMP mirror BAM key file [default]
  -dmpak DMPACCESSKEYPATH, --dmpaccesskeypath DMPACCESSKEYPATH
                        DMP mirror BAM key file for MSK-ACCESS [default]

Rscript reports/create_report.R -h                                      
usage: reports/create_report.R [-h] -t TEMPLATE -p PATIENT_ID -r RESULTS -rc
                               CNA_RESULTS_DIR -tt TUMOR_TYPE -m METADATA
                               [-d DMP_ID] [-ds DMP_SAMPLE_ID] [-dm DMP_MAF]
                               [-o OUTPUT] [-ca] [-pi]

optional arguments:
  -h, --help            show this help message and exit
  -t TEMPLATE, --template TEMPLATE
                        Path to Rmarkdown template file.
  -p PATIENT_ID, --patient-id PATIENT_ID
                        Patient ID
  -r RESULTS, --results RESULTS
                        Path to CSV file containing mutation and genotype
                        results for the patient.
  -rc CNA_RESULTS_DIR, --cna-results-dir CNA_RESULTS_DIR
                        Path to directory containing CNA results for the
                        patient.
  -tt TUMOR_TYPE, --tumor-type TUMOR_TYPE
                        Tumor type
  -m METADATA, --metadata METADATA
                        Path to file containing meta data for each sample.
                        Should contain a 'cmo_sample_id_plasma', 'sex', and
                        'collection_date' columns. Can also optionally include
                        a 'timepoint' column (e.g. for treatment information).
  -d DMP_ID, --dmp-id DMP_ID
                        DMP patient ID (optional).
  -ds DMP_SAMPLE_ID, --dmp-sample-id DMP_SAMPLE_ID
                        DMP sample ID (optional).
  -dm DMP_MAF, --dmp-maf DMP_MAF
                        Path to DMP MAF file (optional).
  -o OUTPUT, --output OUTPUT
                        Output file
  -ca, --combine-access
                        Don't splite VAF plots by clonality.
  -pi, --plot-impact    Also plot VAFs from IMPACT samples.

Rscript R/filter_calls.R -h                                         
usage: R/filter_calls.R [-h] [-m MASTERREF] [-o RESULTSDIR] [-dmpk DMPKEYPATH]
                        [-ch CHLIST] [-c CRITERIA]

optional arguments:
  -h, --help            show this help message and exit
  -m MASTERREF, --masterref MASTERREF
                        File path to master reference file
  -o RESULTSDIR, --resultsdir RESULTSDIR
                        Output directory
  -ch CHLIST, --chlist CHLIST
                        List of signed out CH calls [default]
  -c CRITERIA, --criteria CRITERIA
                        Calling criteria [default]

python csv_to_maf.py  -i /path/to/Test1.csv -i /path/to/Test2.csv -i /path/to/Test3.csv

python csv_to_maf.py  -l /path/to/FileOfFiles.txt

> cat FileOfFiles.txt
/path/to/Test1.csv
/path/to/Test2.csv
/path/to/Test3.csv

python csv_to_maf.py  -n -i /path/to/Test1.csv -i /path/to/Test2.csv -i /path/to/Test3.csv
# OR
python csv_to_maf.py  -n -l /path/to/FileOfFiles.txt

> python csv_to_maf.py --help
Usage: csv_to_maf.py [OPTIONS]

  Tool does the following operations:

  A. Read one or more files from the inputs

  B. Removes unwanted columns, modifying the column headers depending on the
  requirements

  C. Massaging the data frame to make it compatible with MAF format

  D. Write the data frame to a file in MAF format and Excel format

  Requirement: pandas; openpyxl; typing; typer;

Options:
  -l, --list PATH                 File of files, List of CSV files to be
                                  converted to maf, one per line, no header,
                                  CSV file generated by Rscript filter_calls.R
                                  [default: ]

  -i, --csv FILE                  File to convert from csv to maf. CSV file
                                  generated by Rscript filter_calls.R, Can be
                                  given multiple times  [default: ]

  -n, --normal / -N, --keep-normal
                                  Keep samples tagged as normal  [default:
                                  False]

  -p, --prefix TEXT               Prefix of the output MAF and EXCEL file
                                  [default: csv_to_maf_output]

  --install-completion            Install completion for the current shell.
  --show-completion               Show completion for the current shell, to
                                  copy it or customize the installation.

  --help                          Show this message and exit.

python convert_dates_to_days.py -i ./example_input.txt -t2 "SCREEN"

> python convert_dates_to_days.py --help
Usage: convert_dates_to_days.py [OPTIONS]

  Tool to do the following operations: A. Reads meta data file, and based on
  the timepoint information given convert them to days for a samples
  belonging to a given patient_id B. Supports following date formats:
  'MM/DD/YY','M/D/YY','MM/D/YY','M/DD/YY','MM/DD/YYYY','YYYY/MM/DD'

  Requirement: pandas; typer; arrow

Options:
  -i, --input FILE        Input file with the information to convert dates to
                          days  [required]

  -t1, --timepoint1 TEXT  Column name which has timpoint information to use
                          the baseline date, first preference  [default: C1D1]

  -t2, --timepoint2 TEXT  Column name which has timpoint information to use
                          the baseline date, second preference  [default: ]

  -o, --output TEXT       Name of the output file  [default: output.txt]
  --install-completion    Install completion for the current shell.
  --show-completion       Show completion for the current shell, to copy it or
                          customize the installation.

  --help                  Show this message and exit.

access_data_analysis=>0.1.2 # works with this repo tag
typer==0.3.2
typing_extensions==3.10.0.0
pandas==1.2.5
rich==12.1.0

Usage: run_create_report.py [OPTIONS]

Options:
  -r, --repo PATH                 Base path to where the git repository is
                                  located for access_data_analysis

  -s, --script PATH               Path to the create_report.R script, fall
                                  back if `--repo` is not given

  -t, --template PATH             Path to the template.Rmd or
                                  template_days.Rmd to be used with
                                  create_report.R when `--repo` is not given

  -m, --manifest FILE             File containing meta information per sample.
                                  Require following columns in the header:
                                  cmo_patient_id, sample_id, dmp_patient_id,
                                  collection_date or collection_day,
                                  timepoint. If dmp_sample_id column is given
                                  and has information that will be used to run
                                  facets. If dmp_sample_id is not given and
                                  dmp_patient_id is given than it will be used
                                  to get the Tumor sample with lowest number.
                                  If dmp_sample_id or dmp_patient_id is not
                                  given then it will run without the facet maf
                                  file  [required]

  -v, --variant-results DIRECTORY
                                  Base path for all results of small variants
                                  as generated by filter_calls.R script in
                                  access_data_analysis (Make sure only High
                                  Confidence calls are included)  [required]

  -c, --cnv-results DIRECTORY     Base path for all results of CNV as
                                  generated by CNV_processing.R script in
                                  access_data_analysis  [required]

  -f, --facet-repo DIRECTORY      Base path for all results of facets on
                                  Clinical MSK-IMPACT samples  [default: /juno
                                  /work/ccs/shared/resources/impact/facets/all
                                  /]

  -bf, --best-fit                 If this is set to True then we will attempt
                                  to parse `facets_review.manifest` file to
                                  pick the best fit for a given dmp_sample_id
                                  [default: False]

  -l, --tumor-type TEXT           Tumor type label for the report  [required]
  -cfm, --copy-facet-maf          If this is set to True then we will copy the
                                  facet maf file in the directory specified in
                                  `copy_facet_dir`  [default: False]

  -cfd, --copy-facet-dir PATH     Directory path where the facet maf file
                                  should be copied.

  -d, --template-days             If the `--repo` option is specified and if
                                  this is set to True then we will use the
                                  template_days RMarkdown file as the template
                                  [default: False]

  -gm, --generate-markdown        If given, the create_report.R will be run
                                  with `-md` flag to generate markdown
                                  [default: False]

  -ff, --force                    If this is set to True then we will not stop
                                  if an error is encountered in a given sample
                                  while running create_report.R but keep on
                                  running for the next sample  [default:
                                  False]

  --install-completion            Install completion for the current shell.
  --show-completion               Show completion for the current shell, to
                                  copy it or customize the installation.

  --help                          Show this message and exit.

> python python/run_create_report/run_create_report.py \
-m /home/shahr2/bergerlab/Project_10619_D/small_variants/manifest_noDate_days.tsv \
-r /home/shahr2/github/access_data_analysis \
-v /home/shahr2/bergerlab/Project_10619_D/small_variants/results_20Jan2023/results_stringent/ \
-c /home/shahr2/bergerlab/Project_10619_D/small_variants/results_20Jan2023/CNA_final_call_set \
-l "Melanoma" -gm -d -cfm -ff -bf

> python python/run_create_report/run_create_report.py \
-m /home/shahr2/bergerlab/Project_10619_D/small_variants/manifest_noDate_days.tsv \
-r /home/shahr2/github/access_data_analysis \
-v /home/shahr2/bergerlab/Project_10619_D/small_variants/results_20Jan2023/results_stringent/ \
-c /home/shahr2/bergerlab/Project_10619_D/small_variants/results_20Jan2023/CNA_final_call_set \
-l "Melanoma" -gm -ff

def check_required_columns(manifest, template_days=None)

def generate_repo_path(repo_path=None, script_path=None, template_path=None, template_days=None)

def read_manifest(manifest)

def get_row(tsv_file)

def get_small_variant_csv(patient_id, csv_path)

def run_cmd(cmd)

def run_multiple_cmd(commands, parallel_process=None)

def generate_facet_maf_path(facet_path, patient_id, sample_id=None)

def get_maf_path(maf_path, patient_id, sample_id)

def get_best_fit_folder(facet_manifest_path)

def generate_create_report_cmd(script, markdown, template_file, cmo_patient_id, csv_file, manifest, cnv_path, dmp_patient_id, dmp_sample_id, dmp_facet_maf, tumor_type=None)

pip install pandas typer rich arrow numpy openpyxl

python manifest.py make-manifest -i input_manifest.xlsx -o updated_manifest --remove-collection-date -a XS2

python manifest.py update-manifest -i legacy_manifest.xlsx -o updated_legacy_manifest