Add lazyness metadata to plugins

Author: turicas

rows/plugins/dicts.py

@@ -51,6 +51,9 @@ def import_from_dicts(data, samples=1000, *args, **kwargs):
             *args, **kwargs)
 
 
+import_from_dicts.is_lazy = True
+
+
 def export_to_dicts(table, *args, **kwargs):
     return [{key: getattr(row, key) for key in table.field_names}
             for row in table]

rows/plugins/ods.py

@@ -103,5 +103,10 @@ def import_from_ods(filename_or_fobj, index=0, *args, **kwargs):
 
     max_length = max(len(row) for row in table_rows)
     full_rows = complete_with_None(table_rows, max_length)
+
     meta = {'imported_from': 'ods', 'filename': filename,}
+
     return create_table(full_rows, meta=meta, *args, **kwargs)
+
+
+import_from_ods.is_lazy = False

rows/plugins/plugin_csv.py

@@ -119,6 +119,9 @@ def import_from_csv(filename_or_fobj, encoding='utf-8', dialect=None,
     return create_table(reader, meta=meta, *args, **kwargs)
 
 
+import_from_csv.is_lazy = True
+
+
 def export_to_csv(table, filename_or_fobj=None, encoding='utf-8',
                   dialect=unicodecsv.excel, batch_size=100, *args, **kwargs):
     """Export a `rows.Table` to a CSV file

rows/plugins/plugin_html.py

@@ -98,6 +98,9 @@ def import_from_html(filename_or_fobj, encoding='utf-8', index=0,
     return create_table(table_rows, meta=meta, *args, **kwargs)
 
 
+import_from_html.is_lazy = False
+
+
 def export_to_html(table, filename_or_fobj=None, encoding='utf-8', *args,
                    **kwargs):
     serialized_table = serialize(table, *args, **kwargs)
@@ -106,6 +109,7 @@ def export_to_html(table, filename_or_fobj=None, encoding='utf-8', *args,
     header = ['      <th> {} </th>\n'.format(field) for field in fields]
     result.extend(header)
     result.extend(['    </tr>\n', '  </thead>\n', '\n', '  <tbody>\n', '\n'])
+    # TODO: could be lazy so we don't need to store the whole table into memory
     for index, row in enumerate(serialized_table, start=1):
         css_class = 'odd' if index % 2 == 1 else 'even'
         result.append('    <tr class="{}">\n'.format(css_class))

rows/plugins/plugin_json.py

@@ -38,6 +38,7 @@ def import_from_json(filename_or_fobj, encoding='utf-8', *args, **kwargs):
     filename, fobj = get_filename_and_fobj(filename_or_fobj)
 
     json_obj = json.load(fobj, encoding=encoding)
+    # TODO: may use import_from_dicts here
     field_names = list(json_obj[0].keys())
     table_rows = [[item[key] for key in field_names] for item in json_obj]
 
@@ -47,6 +48,9 @@ def import_from_json(filename_or_fobj, encoding='utf-8', *args, **kwargs):
     return create_table([field_names] + table_rows, meta=meta, *args, **kwargs)
 
 
+import_from_json.is_lazy = False
+
+
 def _convert(value, field_type, *args, **kwargs):
     if value is None or field_type in (
                 fields.BinaryField,
@@ -77,6 +81,8 @@ def export_to_json(table, filename_or_fobj=None, encoding='utf-8', indent=None,
     fields = table.fields
     prepared_table = prepare_to_export(table, *args, **kwargs)
     field_names = next(prepared_table)
+
+    # TODO: could be lazy so we don't need to store the whole table into memory
     data = [{field_name: _convert(value, fields[field_name], *args, **kwargs)
              for field_name, value in zip(field_names, row)}
             for row in prepared_table]

rows/plugins/plugin_parquet.py

@@ -56,8 +56,12 @@ def import_from_parquet(filename_or_fobj, *args, **kwargs):
                          for schema in parquet._read_footer(fobj).schema
                          if schema.type is not None])
     header = list(types.keys())
-    table_rows = list(parquet.reader(fobj))  # TODO: be lazy
+    # TODO: make it lazy
+    table_rows = list(parquet.reader(fobj))
 
     meta = {'imported_from': 'parquet', 'filename': filename,}
     return create_table([header] + table_rows, meta=meta, force_types=types,
                         *args, **kwargs)
+
+
+import_from_parquet.is_lazy = False

rows/plugins/sqlite.py

@@ -128,6 +128,9 @@ def import_from_sqlite(filename_or_connection, table_name='table1', query=None,
     return create_table(data, meta=meta, *args, **kwargs)
 
 
+import_from_sqlite.is_lazy = True
+
+
 def export_to_sqlite(table, filename_or_connection, table_name=None,
                      table_name_format='table{index}', batch_size=100,
                      *args, **kwargs):

rows/plugins/txt.py

@@ -34,18 +34,24 @@ def import_from_txt(filename_or_fobj, encoding='utf-8', *args, **kwargs):
     filename, fobj = get_filename_and_fobj(filename_or_fobj, mode='rb')
     contents = fobj.read().decode(encoding).strip().splitlines()
 
+    # TODO: make it lazy
+
     # remove '+----+----+' lines
     contents = contents[1:-1]
     del contents[1]
 
     table_rows = [[value.strip() for value in row.split(PIPE)[1:-1]]
                   for row in contents]
+
     meta = {'imported_from': 'txt',
             'filename': filename,
             'encoding': encoding,}
     return create_table(table_rows, meta=meta, *args, **kwargs)
 
 
+import_from_txt.is_lazy = False
+
+
 def export_to_txt(table, filename_or_fobj=None, encoding=None,
                   *args, **kwargs):
     '''Export a `rows.Table` to text
@@ -70,6 +76,7 @@ def export_to_txt(table, filename_or_fobj=None, encoding=None,
     split_line = PLUS + PLUS.join(dashes) + PLUS
 
     result = [split_line, header, split_line]
+    # TODO: make it lazy
     for row in table_rows:
         values = [value.rjust(max_sizes[field_name])
                   for field_name, value in zip(field_names, row)]

rows/plugins/utils.py

@@ -141,7 +141,10 @@ def func(rows_data):
 def create_table(data, meta=None, fields=None, skip_header=True,
                  import_fields=None, samples=None, force_types=None,
                  lazy=False, *args, **kwargs):
+    # TODO: change samples to be a fixed number
+    # TODO: may change samples logic (`float('inf')` or `all`)
     # TODO: add auto_detect_types=True parameter
+
     table_rows = iter(data)
     sample_rows = []
 
@@ -163,6 +166,9 @@ def create_table(data, meta=None, fields=None, skip_header=True,
         if not isinstance(fields, OrderedDict):
             raise ValueError('`fields` must be an `OrderedDict`')
 
+        # TODO: if `fields` is set, we're going to have the wrong order,
+        # compared to the first row (header).
+
         if skip_header:
             _ = next(table_rows)
 
@@ -187,6 +193,7 @@ def create_table(data, meta=None, fields=None, skip_header=True,
 
     if not lazy:
         table = Table(fields=fields, meta=meta)
+
         # TODO: put this inside Table.__init__
         for row in chain(sample_rows, table_rows):
             table.append({field_name: value

rows/plugins/xls.py

@@ -156,6 +156,9 @@ def import_from_xls(filename_or_fobj, sheet_name=None, sheet_index=0,
     return create_table(table_rows, meta=meta, *args, **kwargs)
 
 
+import_from_xls.is_lazy = False
+
+
 def export_to_xls(table, filename_or_fobj=None, sheet_name='Sheet1', *args,
                   **kwargs):