Skip to content

Use newer viame runners for training and inference as opposed to kwiver call #1574

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mattdawkins wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Use newer viame runners for training and inference as opposed to kwiver call #1574

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions client/platform/desktop/backend/native/linux.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ const DefaultSettings: Settings = {

const ViameLinuxConstants = {
setup: 'setup_viame.sh',
trainingExe: 'viame_train_detector',
trainingExe: 'viame',
kwiverExe: 'kwiver',
shell: '/bin/bash',
};
Expand All @@ -57,10 +57,10 @@ async function validateViamePath(settings: Settings): Promise<true | string> {
return `${setupScriptPath} does not exist`;
}

const trainingScriptPath = npath.join(settings.viamePath, 'bin', ViameLinuxConstants.trainingExe);
const trainingExists = await fs.pathExists(trainingScriptPath);
if (!trainingExists) {
return `${trainingScriptPath} does not exist`;
const viameExePath = npath.join(settings.viamePath, 'bin', ViameLinuxConstants.trainingExe);
const viameExists = await fs.pathExists(viameExePath);
if (!viameExists) {
return `${viameExePath} does not exist`;
}

const kwiverExistsOnPath = observeChild(spawn(
Expand Down
9 changes: 5 additions & 4 deletions client/platform/desktop/backend/native/viame.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ async function runPipeline(
}
command = [
`${viameConstants.setupScriptAbs} &&`,
`"${viameConstants.kwiverExe}" runner`,
`"${viameConstants.trainingExe}" runner`,
'-s "input:video_reader:type=vidl_ffmpeg"',
`-p "${pipelinePath}"`,
`-s downsampler:target_frame_rate=${meta.fps}`,
Expand All @@ -119,7 +119,7 @@ async function runPipeline(
await fs.writeFile(manifestFile, fileData);
command = [
`${viameConstants.setupScriptAbs} &&`,
`"${viameConstants.kwiverExe}" runner`,
`"${viameConstants.trainingExe}" runner`,
`-p "${pipelinePath}"`,
];
if (!stereoOrMultiCam) {
Expand Down Expand Up @@ -251,7 +251,8 @@ async function exportTrainedPipeline(

const command = [
`${viameConstants.setupScriptAbs} &&`,
`"${viameConstants.kwiverExe}" runner ${exportPipelinePath}`,
`"${viameConstants.trainingExe}" runner`,
`-p "${exportPipelinePath}"`,
`-s "onnx_convert:model_path=${weightsPath}"`,
`-s "onnx_convert:onnx_model_prefix=${converterOutput}"`,
];
Expand Down Expand Up @@ -382,7 +383,7 @@ async function train(

const command = [
`${viameConstants.setupScriptAbs} &&`,
`"${viameConstants.trainingExe}"`,
`"${viameConstants.trainingExe}" train`,
`--input-list "${inputFolderFileList}"`,
`--input-truth "${groundTruthFileList}"`,
`--config "${configFilePath}"`,
Expand Down
2 changes: 1 addition & 1 deletion client/platform/desktop/backend/native/windows.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ const DefaultSettings: Settings = {

const ViameWindowsConstants = {
setup: 'setup_viame.bat',
trainingExe: 'viame_train_detector.exe',
trainingExe: 'viame.exe',
kwiverExe: 'kwiver.exe',
shell: true,
};
Expand Down
2 changes: 1 addition & 1 deletion docs/Command-Line-Tools.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

!!! note

This page is **not related** to the VIAME command line (i.e. `kwiver`, `viame_train_detector`)
This page is **not related** to the VIAME command line (i.e. `kwiver`, `viame`)

Some of the DIVE data conversion features are exposed through `dive`.

Expand Down
18 changes: 9 additions & 9 deletions server/dive_tasks/tasks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,8 @@ def __init__(self):
assert self.viame_install_path.exists(), "VIAME Base install directory missing."
self.viame_setup_script = self.viame_install_path / "setup_viame.sh"
assert self.viame_setup_script.is_file(), "VIAME Setup Script missing"
self.viame_training_executable = self.viame_install_path / "bin" / "viame_train_detector"
assert self.viame_training_executable.is_file(), "VIAME Training Executable missing"
self.viame_training_executable = self.viame_install_path / "bin" / "viame"
assert self.viame_training_executable.is_file(), "VIAME Executable missing"

# The subdirectory within VIAME_INSTALL_PATH where pipelines can be found
self.pipeline_subdir = 'configs/pipelines'
Expand Down Expand Up @@ -232,7 +232,7 @@ def run_pipeline(self: Task, params: PipelineJob):
command = [
f". {shlex.quote(str(conf.viame_setup_script))} &&",
f"KWIVER_DEFAULT_LOG_LEVEL={shlex.quote(conf.kwiver_log_level)}",
"kwiver runner",
"viame runner",
"-s input:video_reader:type=vidl_ffmpeg",
f"-p {shlex.quote(str(pipeline_path))}",
f"-s input:video_filename={shlex.quote(input_media_list[0])}",
Expand All @@ -246,7 +246,7 @@ def run_pipeline(self: Task, params: PipelineJob):
command = [
f". {shlex.quote(str(conf.viame_setup_script))} &&",
f"KWIVER_DEFAULT_LOG_LEVEL={shlex.quote(conf.kwiver_log_level)}",
"kwiver runner",
"viame runner",
f"-p {shlex.quote(str(pipeline_path))}",
f"-s input:video_filename={shlex.quote(str(img_list_path))}",
f"-s detector_writer:file_name={shlex.quote(detector_output_file)}",
Expand Down Expand Up @@ -316,8 +316,8 @@ def export_trained_pipeline(self: Task, params: ExportTrainedPipelineJob):
command = [
f". {shlex.quote(str(conf.viame_setup_script))} &&",
f"KWIVER_DEFAULT_LOG_LEVEL={shlex.quote(conf.kwiver_log_level)}",
"kwiver runner",
f"{shlex.quote(str(convert_to_onnx_pipeline_path))}",
"viame runner",
f"-p {shlex.quote(str(convert_to_onnx_pipeline_path))}",
f"-s onnx_convert:model_path={shlex.quote(str(trained_pipeline_path / 'yolo.weights'))}",
f"-s onnx_convert:onnx_model_prefix={shlex.quote(str(onnx_path))}"
]
Expand All @@ -338,7 +338,7 @@ def export_trained_pipeline(self: Task, params: ExportTrainedPipelineJob):

@app.task(bind=True, acks_late=True, ignore_result=True)
def train_pipeline(self: Task, params: TrainingJob):
"""Train a pipeline by making a call to viame_train_detector"""
"""Train a pipeline by making a call to viame train"""
conf = Config()
context: dict = {}
manager: JobManager = patch_manager(self.job_manager)
Expand All @@ -364,7 +364,7 @@ def train_pipeline(self: Task, params: TrainingJob):
config_file = pipeline_base_path / config
# List of (input folder, ground truth file) pairs for creating input lists
input_groundtruth_list: List[Tuple[Path, Path]] = []
# root_data_dir is the directory passed to `viame_train_detector`
# root_data_dir is the directory passed to `viame train`
with tempfile.TemporaryDirectory() as _working_directory, suppress(utils.CanceledError):
_working_directory_path = Path(_working_directory)
input_path = utils.make_directory(_working_directory_path / 'input')
Expand Down Expand Up @@ -397,7 +397,7 @@ def train_pipeline(self: Task, params: TrainingJob):
command = [
f". {shlex.quote(str(conf.viame_setup_script))} &&",
f"KWIVER_DEFAULT_LOG_LEVEL={shlex.quote(conf.kwiver_log_level)}",
shlex.quote(str(conf.viame_training_executable)),
f"{shlex.quote(str(conf.viame_training_executable))} train",
"--input-list",
shlex.quote(str(input_folder_file_list)),
"--input-truth",
Expand Down
Loading