[EGGO-30] Add support for partitioning BAM/SAM.

Author: tomwhite

eggo/dag.py

@@ -309,6 +309,7 @@ class ADAMPartitionTask(Task):
 
     adam_command = Parameter()
     allowed_file_formats = Parameter()
+    partition_strategy_file = Parameter()
     source_edition = 'basic'
     edition = 'locuspart'
 
@@ -323,7 +324,7 @@ def run(self):
             hadoop_home=os.environ['HADOOP_HOME'],
             adam_partitioning_jar=os.environ['ADAM_PARTITIONING_JAR'],
             parallelism=1,
-            partition_strategy_file='genotypes-partition-strategy',
+            partition_strategy_file=self.partition_strategy_file,
             source=target_s3n_url(ToastConfig().config['name'],
                                   edition=self.source_edition),
             target=target_s3n_url(ToastConfig().config['name'],
@@ -344,6 +345,7 @@ class ADAMFlattenPartitionTask(Task):
 
     adam_command = Parameter()
     allowed_file_formats = Parameter()
+    partition_strategy_file = Parameter()
     source_edition = 'flat'
     edition = 'flat_locuspart'
 
@@ -358,7 +360,7 @@ def run(self):
             hadoop_home=os.environ['HADOOP_HOME'],
             adam_partitioning_jar=os.environ['ADAM_PARTITIONING_JAR'],
             parallelism=1,
-            partition_strategy_file='flat-genotypes-partition-strategy',
+            partition_strategy_file=self.partition_strategy_file,
             source=target_s3n_url(ToastConfig().config['name'],
                                   edition=self.source_edition),
             target=target_s3n_url(ToastConfig().config['name'],
@@ -383,9 +385,11 @@ def requires(self):
         flat = ADAMFlattenTask(adam_command='vcf2adam',
                                allowed_file_formats=['vcf'])
         locuspart = ADAMPartitionTask(adam_command='vcf2adam',
-                                      allowed_file_formats=['vcf'])
+                                      allowed_file_formats=['vcf'],
+                                      partition_strategy_file='genotypes-partition-strategy')
         flat_locuspart = ADAMFlattenPartitionTask(adam_command='vcf2adam',
-                                                  allowed_file_formats=['vcf'])
+                                                  allowed_file_formats=['vcf'],
+                                                  partition_strategy_file='flat-genotypes-partition-strategy')
         dependencies = [basic]
         for edition in ToastConfig().config['editions']:
             if edition == 'basic':
@@ -413,9 +417,11 @@ def requires(self):
         flat = ADAMFlattenTask(adam_command='transform',
                                allowed_file_formats=['sam', 'bam'])
         locuspart = ADAMPartitionTask(adam_command='transform',
-                                      allowed_file_formats=['sam', 'bam'])
+                                      allowed_file_formats=['sam', 'bam'],
+                                      partition_strategy_file='alignments-partition-strategy')
         flat_locuspart = ADAMFlattenPartitionTask(adam_command='transform',
-                                                  allowed_file_formats=['sam', 'bam'])
+                                                  allowed_file_formats=['sam', 'bam'],
+                                                  partition_strategy_file='flat-alignments-partition-strategy')
         dependencies = [basic]
         for edition in ToastConfig().config['editions']:
             if edition == 'basic':

genotypes-flat-partition-strategy.json

@@ -0,0 +1,9 @@
+{
+    "name": "test-1kg-genotypes-subset",
+    "title": "Test 1000 Genomes Project VCF data",
+    "dag": "VCF2ADAMTask",
+    "editions": ["basic", "flat", "locuspart", "flat_locuspart"],
+    "sources": [
+      {"format": "vcf", "compression": true, "url": "ftp://ftp-trace.ncbi.nih.gov/1000genomes/ftp/release/20110521/ALL.chr22.phase1_release_v3.20101123.snps_indels_svs.genotypes.vcf.gz"}
+    ]
+}

genotypes-partition-strategy.json

@@ -2,8 +2,8 @@
     "name": "test-alignments",
     "title": "Test SAM data",
     "dag": "BAM2ADAMTask",
-    "editions": ["basic", "flat"],
+    "editions": ["basic", "flat", "locuspart", "flat_locuspart"],
     "sources": [
-        {"format": "sam", "compression": false, "url": "https://raw.githubusercontent.com/bigdatagenomics/adam/master/adam-core/src/test/resources/reads12.sam"}
+        {"format": "sam", "compression": false, "url": "https://github.com/bigdatagenomics/eggo/raw/master/test/resources/small.sam"}
     ]
 }