Reorg, incl splitting up util and adding ADAM variant conversion

Also moved old code to a temporary DELETE_ME dir

Author: laserson

eggo/dag.py renamed to DELETE_ME/dag.py

@@ -16,16 +16,17 @@
 # limitations under the License.
 
 import json
-import os
 import os.path as osp
 from os.path import join as pjoin
 
-from eggo.datasets import download_dataset_with_hadoop
+from eggo.datasets import download_dataset_with_hadoop, vcf_to_adam_variants
 
 
-hdfs_path = '/user/ec2-user/dbsnp/raw'
+raw_data_path = '/user/ec2-user/dbsnp/raw'
+adam_variants_path = '/user/ec2-user/dbsnp/variants/adam/basic'
 
 with open(pjoin(osp.dirname(__file__), 'datapackage.json')) as ip:
     datapackage = json.load(ip)
 
-download_dataset_with_hadoop(datapackage, hdfs_path)
+download_dataset_with_hadoop(datapackage, raw_data_path)
+vcf_to_adam_variants(raw_data_path, adam_variants_path)

eggo/fabric_cli.py renamed to DELETE_ME/fabric_cli.py

@@ -14,11 +14,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os.path as osp
+
 from click import group, option, Choice
+from fabric.api import execute, get
 
 import eggo.cluster.director as director
-from eggo.cluster.config import (
-    DEFAULT_DIRECTOR_CONF_PATH, DEFAULT_CF_TEMPLATE_PATH)
+
+
+DEFAULT_DIRECTOR_CONF_PATH = osp.join(
+    osp.dirname(__file__), 'resources', 'aws.conf')
+DEFAULT_CF_TEMPLATE_PATH = osp.join(
+    osp.dirname(__file__), 'resources', 'cloudformation.template')
 
 
 # reusable options
@@ -93,7 +100,7 @@ def login(region, stack_name, node):
 @option_stack_name
 def describe(region, stack_name):
     """Describe the EC2 instances in the cluster"""
-    director.list(region, stack_name)
+    director.describe(region, stack_name)
 
 
 @cli.command()
@@ -109,7 +116,6 @@ def web_proxy(region, stack_name):
 @option_stack_name
 def get_director_log(region, stack_name):
     """DEBUG: get the Director application log from the launcher instance"""
-    from fabric.api import execute, get
     ec2_conn = director.create_ec2_connection(region)
     hosts = [director.get_launcher_instance(ec2_conn, stack_name).ip_address]
     execute(
@@ -124,7 +130,6 @@ def get_director_log(region, stack_name):
 @option('-b', '--branch', default='master', show_default=True)
 def reinstall_eggo(region, stack_name, fork, branch):
     """DEBUG: reinstall a specific version of eggo"""
-    from fabric.api import execute
     ec2_conn = director.create_ec2_connection(region)
     hosts = [director.get_master_instance(ec2_conn, stack_name).ip_address]
     execute(