Port row_diff_traverse from DBGSuccinct

.github/workflows/main.yml

@@ -7,8 +7,6 @@ on:
     tags:
       - 'v*'
   pull_request:
-    branches:
-      - master
 
 env:
   REGISTRY: ghcr.io

metagraph/CMakeLists.txt

@@ -111,6 +111,7 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
   add_compile_options(
     -Wno-exit-time-destructors
     -Wno-deprecated-declarations
+    -Wno-vla-extension
   )
 
   if (NOT CMAKE_CXX_COMPILER_ID MATCHES "AppleClang")

metagraph/integration_tests/base.py

@@ -6,7 +6,14 @@
 
 script_path = os.path.dirname(os.path.realpath(__file__))
 
-METAGRAPH = f'{os.getcwd()}/metagraph'
+METAGRAPH_EXE = f'{os.getcwd()}/metagraph'
+DNA_MODE = os.readlink(METAGRAPH_EXE).endswith("_DNA")
+PROTEIN_MODE = os.readlink(METAGRAPH_EXE).endswith("_Protein")
+METAGRAPH = METAGRAPH_EXE
+
+def update_prefix(PREFIX):
+    global METAGRAPH
+    METAGRAPH = PREFIX + METAGRAPH_EXE
 
 TEST_DATA_DIR = os.path.join(script_path, '..', 'tests', 'data')
 
@@ -37,10 +44,19 @@ def setUpClass(cls):
     def _get_stats(graph_path):
         stats_command = METAGRAPH + ' stats ' + graph_path + ' --mmap'
         res = subprocess.run(stats_command.split(), stdout=PIPE, stderr=PIPE)
-        assert(res.returncode == 0)
+        if res.returncode != 0:
+            raise AssertionError(f"Command '{stats_command}' failed with return code {res.returncode} and error: {res.stderr.decode()}")
         stats_command = METAGRAPH + ' stats ' + graph_path + MMAP_FLAG
         res = subprocess.run(stats_command.split(), stdout=PIPE, stderr=PIPE)
-        return res
+        parsed = dict()
+        parsed['returncode'] = res.returncode
+        res = res.stdout.decode().split('\n')[2:]
+        for line in res:
+            if ': ' in line:
+                x, y = map(str.strip, line.split(':', 1))
+                assert(x not in parsed or parsed[x] == y)
+                parsed[x] = y
+        return parsed
 
     @staticmethod
     def _build_graph(input, output, k, repr, mode='basic', extra_params=''):

metagraph/integration_tests/main.py

@@ -5,6 +5,7 @@
 import sys
 import argparse
 from helpers import TimeLoggingTestResult
+from base import update_prefix
 
 
 """Run all integration tests"""
@@ -32,7 +33,11 @@ def create_test_suite(filter_pattern="*"):
         parser = argparse.ArgumentParser(description='Metagraph integration tests.')
         parser.add_argument('--test_filter', dest='filter', type=str, default="*",
                             help='filter test cases (default: run all)')
+        parser.add_argument('--gdb', dest='use_gdb', action='store_true',
+                            help='run metagraph with gdb')
         args = parser.parse_args()
+        if args.use_gdb:
+            update_prefix('gdb -ex run -ex bt -ex quit --args ')
 
         result = unittest.TextTestRunner(
             resultclass=TimeLoggingTestResult

metagraph/integration_tests/test_align.py

@@ -6,14 +6,11 @@
 import glob
 import os
 
-from base import TestingBase, METAGRAPH, TEST_DATA_DIR, NUM_THREADS
+from base import PROTEIN_MODE, DNA_MODE, TestingBase, METAGRAPH, TEST_DATA_DIR, NUM_THREADS
 
 
 """Test graph construction and alignment"""
 
-DNA_MODE = os.readlink(METAGRAPH).endswith("_DNA")
-PROTEIN_MODE = os.readlink(METAGRAPH).endswith("_Protein")
-
 graph_file_extension = {'succinct': '.dbg',
                         'bitmap': '.bitmapdbg',
                         'hash': '.orhashdbg',
@@ -35,11 +32,10 @@ def test_simple_align_all_graphs(self, representation):
                           k=11, repr=representation,
                           extra_params="--mask-dummy")
 
-        res = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
-        params_str = res.stdout.decode().split('\n')[2:]
-        self.assertEqual('k: 11', params_str[0])
-        self.assertEqual('nodes (k): 16438', params_str[1])
-        self.assertEqual('mode: basic', params_str[2])
+        params = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
+        self.assertEqual('11', params['k'])
+        self.assertEqual('16438', params['nodes (k)'])
+        self.assertEqual('basic', params['mode'])
 
         stats_command = '{exe} align --align-only-forwards -i {graph} --align-min-exact-match 0.0 {reads}'.format(
             exe=METAGRAPH,
@@ -68,11 +64,10 @@ def test_simple_align_map_all_graphs(self, representation):
                           k=11, repr=representation,
                           extra_params="--mask-dummy")
 
-        res = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
-        params_str = res.stdout.decode().split('\n')[2:]
-        self.assertEqual('k: 11', params_str[0])
-        self.assertEqual('nodes (k): 16438', params_str[1])
-        self.assertEqual('mode: basic', params_str[2])
+        params = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
+        self.assertEqual('11', params['k'])
+        self.assertEqual('16438', params['nodes (k)'])
+        self.assertEqual('basic', params['mode'])
 
         stats_command = '{exe} align -i {graph} --map --count-kmers {reads}'.format(
             exe=METAGRAPH,
@@ -99,11 +94,10 @@ def test_simple_align_map_all_graphs_subk(self, representation):
                           k=11, repr=representation,
                           extra_params="--mask-dummy")
 
-        res = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
-        params_str = res.stdout.decode().split('\n')[2:]
-        self.assertEqual('k: 11', params_str[0])
-        self.assertEqual('nodes (k): 16438', params_str[1])
-        self.assertEqual('mode: basic', params_str[2])
+        params = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
+        self.assertEqual('11', params['k'])
+        self.assertEqual('16438', params['nodes (k)'])
+        self.assertEqual('basic', params['mode'])
 
         stats_command = '{exe} align -i {graph} --map --count-kmers --align-length 10 {reads}'.format(
             exe=METAGRAPH,
@@ -134,11 +128,10 @@ def test_simple_align_map_canonical_all_graphs(self, representation):
                           k=11, repr=representation, mode='canonical',
                           extra_params="--mask-dummy")
 
-        res = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
-        params_str = res.stdout.decode().split('\n')[2:]
-        self.assertEqual('k: 11', params_str[0])
-        self.assertEqual('nodes (k): 32782', params_str[1])
-        self.assertEqual('mode: canonical', params_str[2])
+        params = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
+        self.assertEqual('11', params['k'])
+        self.assertEqual('32782', params['nodes (k)'])
+        self.assertEqual('canonical', params['mode'])
 
         stats_command = '{exe} align -i {graph} --map --count-kmers {reads}'.format(
             exe=METAGRAPH,
@@ -165,11 +158,10 @@ def test_simple_align_json_all_graphs(self, representation):
                           k=11, repr=representation,
                           extra_params="--mask-dummy")
 
-        res = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
-        params_str = res.stdout.decode().split('\n')[2:]
-        self.assertEqual('k: 11', params_str[0])
-        self.assertEqual('nodes (k): 16438', params_str[1])
-        self.assertEqual('mode: basic', params_str[2])
+        params = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
+        self.assertEqual('11', params['k'])
+        self.assertEqual('16438', params['nodes (k)'])
+        self.assertEqual('basic', params['mode'])
 
         stats_command = '{exe} align --align-only-forwards -i {graph} --align-min-exact-match 0.0 {reads}'.format(
             exe=METAGRAPH,
@@ -189,11 +181,10 @@ def test_simple_align_fwd_rev_comp_all_graphs(self, representation):
                           k=11, repr=representation,
                           extra_params="--mask-dummy")
 
-        res = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
-        params_str = res.stdout.decode().split('\n')[2:]
-        self.assertEqual('k: 11', params_str[0])
-        self.assertEqual('nodes (k): 16438', params_str[1])
-        self.assertEqual('mode: basic', params_str[2])
+        params = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
+        self.assertEqual('11', params['k'])
+        self.assertEqual('16438', params['nodes (k)'])
+        self.assertEqual('basic', params['mode'])
 
         stats_command = '{exe} align -i {graph} --align-min-exact-match 0.0 {reads}'.format(
             exe=METAGRAPH,
@@ -222,11 +213,10 @@ def test_simple_align_canonical_all_graphs(self, representation):
                           k=11, repr=representation, mode='canonical',
                           extra_params="--mask-dummy")
 
-        res = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
-        params_str = res.stdout.decode().split('\n')[2:]
-        self.assertEqual('k: 11', params_str[0])
-        self.assertEqual('nodes (k): 32782', params_str[1])
-        self.assertEqual('mode: canonical', params_str[2])
+        params = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
+        self.assertEqual('11', params['k'])
+        self.assertEqual('32782', params['nodes (k)'])
+        self.assertEqual('canonical', params['mode'])
 
         stats_command = '{exe} align -i {graph} --align-min-exact-match 0.0 {reads}'.format(
             exe=METAGRAPH,
@@ -256,11 +246,10 @@ def test_simple_align_canonical_subk_succinct(self, representation):
                           k=11, repr=representation, mode='canonical',
                           extra_params="--mask-dummy")
 
-        res = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
-        params_str = res.stdout.decode().split('\n')[2:]
-        self.assertEqual('k: 11', params_str[0])
-        self.assertEqual('nodes (k): 32782', params_str[1])
-        self.assertEqual('mode: canonical', params_str[2])
+        params = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
+        self.assertEqual('11', params['k'])
+        self.assertEqual('32782', params['nodes (k)'])
+        self.assertEqual('canonical', params['mode'])
 
         stats_command = '{exe} align -i {graph} --align-min-exact-match 0.0 --align-min-seed-length 10 {reads}'.format(
             exe=METAGRAPH,
@@ -286,11 +275,10 @@ def test_simple_align_primary_all_graphs(self, representation):
                           k=11, repr=representation, mode='primary',
                           extra_params="--mask-dummy")
 
-        res = self._get_stats(self.tempdir.name + '/genome.MT.primary' + graph_file_extension[representation])
-        params_str = res.stdout.decode().split('\n')[2:]
-        self.assertEqual('k: 11', params_str[0])
-        self.assertEqual('nodes (k): 16391', params_str[1])
-        self.assertEqual('mode: primary', params_str[2])
+        params = self._get_stats(self.tempdir.name + '/genome.MT.primary' + graph_file_extension[representation])
+        self.assertEqual('11', params['k'])
+        self.assertEqual('16391', params['nodes (k)'])
+        self.assertEqual('primary', params['mode'])
 
         stats_command = '{exe} align -i {graph} --align-min-exact-match 0.0 {reads}'.format(
             exe=METAGRAPH,
@@ -320,11 +308,10 @@ def test_simple_align_primary_subk_succinct(self, representation):
                           k=11, repr=representation, mode='primary',
                           extra_params="--mask-dummy")
 
-        res = self._get_stats(self.tempdir.name + '/genome.MT.primary' + graph_file_extension[representation])
-        params_str = res.stdout.decode().split('\n')[2:]
-        self.assertEqual('k: 11', params_str[0])
-        self.assertEqual('nodes (k): 16391', params_str[1])
-        self.assertEqual('mode: primary', params_str[2])
+        params = self._get_stats(self.tempdir.name + '/genome.MT.primary' + graph_file_extension[representation])
+        self.assertEqual('11', params['k'])
+        self.assertEqual('16391', params['nodes (k)'])
+        self.assertEqual('primary', params['mode'])
 
         stats_command = '{exe} align -i {graph} --align-min-exact-match 0.0 --align-min-seed-length 10 {reads}'.format(
             exe=METAGRAPH,
@@ -347,13 +334,13 @@ def test_simple_align_fwd_rev_comp_json_all_graphs(self, representation):
 
         self._build_graph(input=TEST_DATA_DIR + '/genome.MT.fa',
                           output=self.tempdir.name + '/genome.MT',
-                          k=11, repr=representation)
+                          k=11, repr=representation,
+                          extra_params="--mask-dummy")
 
-        res = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
-        params_str = res.stdout.decode().split('\n')[2:]
-        self.assertEqual('k: 11', params_str[0])
-        self.assertEqual('nodes (k): 16461', params_str[1])
-        self.assertEqual('mode: basic', params_str[2])
+        params = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
+        self.assertEqual('11', params['k'])
+        self.assertEqual('16438', params['nodes (k)'])
+        self.assertEqual('basic', params['mode'])
 
         stats_command = '{exe} align --json -i {graph} --align-min-exact-match 0.0 {reads}'.format(
             exe=METAGRAPH,
@@ -373,13 +360,13 @@ def test_simple_align_edit_distance_all_graphs(self, representation):
 
         self._build_graph(input=TEST_DATA_DIR + '/genome.MT.fa',
                           output=self.tempdir.name + '/genome.MT',
-                          k=11, repr=representation)
+                          k=11, repr=representation,
+                          extra_params="--mask-dummy")
 
-        res = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
-        params_str = res.stdout.decode().split('\n')[2:]
-        self.assertEqual('k: 11', params_str[0])
-        self.assertEqual('nodes (k): 16461', params_str[1])
-        self.assertEqual('mode: basic', params_str[2])
+        params = self._get_stats(self.tempdir.name + '/genome.MT' + graph_file_extension[representation])
+        self.assertEqual('11', params['k'])
+        self.assertEqual('16438', params['nodes (k)'])
+        self.assertEqual('basic', params['mode'])
 
         stats_command = '{exe} align --json --align-edit-distance -i {graph} --align-min-exact-match 0.0 {reads}'.format(
             exe=METAGRAPH,