diff --git a/.gitignore b/.gitignore index e39c48d..3081626 100644 --- a/.gitignore +++ b/.gitignore @@ -164,3 +164,4 @@ data/ converted_data/ db/ old_2024_data/ +imported_files/ diff --git a/ingest_library.py b/ingest_library.py index bb0322d..51793e8 100644 --- a/ingest_library.py +++ b/ingest_library.py @@ -33,7 +33,7 @@ def update_file_metadata(connection, filename, hash, success): connection.rollback() def read_device_logfile(filepath): - if 'rio' in os.basename(filepath): + if 'rio' in os.path.basename(filepath): convert_folder = 'converted_rio_device_logs' else: convert_folder = 'converted_drive_device_logs' diff --git a/manage_imports.py b/manage_imports.py index 47094a1..14d33f6 100644 --- a/manage_imports.py +++ b/manage_imports.py @@ -1,29 +1,38 @@ import os import shutil +import sys + + +from ingest_match_logs import ingest_match_logs def manage_imports(): print("Managing imports") import_list = os.listdir("./imported_files") - + import_system_list = [] import_drive_list = [] import_rio_list = [] - + + os.makedirs("./data", exist_ok=True) + os.makedirs("./data/system_logs", exist_ok = True) + os.makedirs("./data/rio_device_logs", exist_ok = True) + os.makedirs("./data/drive_device_logs", exist_ok = True) + for file in import_list: pos = file.rfind(".") - if file[pos:] is "wpilog": + if file[pos:] == ".wpilog": import_system_list.append(file) - shutil.copy("./data/imported_files/" + file, "./data/system_logs") - elif file[pos:] is "hoot": + shutil.copy("./imported_files/" + file, "./data/system_logs") + elif file[pos:] == ".hoot": if "rio" in file: import_rio_list.append(file) - shutil.copy("./data/imported_files/" + file, "./data/rio_device_logs") + shutil.copy("./imported_files/" + file, "./data/rio_device_logs") else: import_drive_list.append(file) - shutil.copy("./data/imported_files/" + file, "./data/drive_device_logs") + shutil.copy("./imported_files/" + file, "./data/drive_device_logs") else: print("Unrecognized filetype is in import directory!") - os.remove("./imported_files/" + file) - + #os.remove("./imported_files/" + file) + #TODO handle exceptions ingest_dict = {} for file in import_system_list: @@ -35,8 +44,13 @@ def manage_imports(): for file in import_rio_list: match_id = file.split("_")[1] ingest_dict[match_id].append("./data/rio_device_logs/"+file) - + return ingest_dict - + if __name__ == '__main__': - manage_imports() \ No newline at end of file + if len(sys.argv) != 2: + print(f"Usage: {sys.argv[0]} ", file=sys.stderr) + sys.exit(1) + ingest_dict = manage_imports() + for match in list(ingest_dict.values()): + ingest_match_logs(match[0], match[1], match[2],sys.argv[1]) \ No newline at end of file