diff --git a/.readthedocs.yml b/.readthedocs.yml index 86e746457..0dcb8d871 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -8,7 +8,7 @@ version: 2 build: os: "ubuntu-20.04" tools: - python: "3.7" + python: "3.9" # Build documentation in the docs/ directory with Sphinx sphinx: @@ -20,3 +20,4 @@ python: path: . extra_requirements: - doc + - reports diff --git a/phys2bids/cli/run.py b/phys2bids/cli/run.py index a7770171e..50da14e5a 100644 --- a/phys2bids/cli/run.py +++ b/phys2bids/cli/run.py @@ -189,6 +189,15 @@ def _get_parser(): help="Only print warnings to log file. Default is False.", default=False, ) + optional.add_argument( + "-report", + "--report", + dest="make_report", + action="store_true", + help="Generate a report with the data and generated folder structure. " + "Default is False.", + default=False, + ) optional.add_argument("-v", "--version", action="version", version=("%(prog)s " + __version__)) parser._action_groups.append(optional) diff --git a/phys2bids/phys2bids.py b/phys2bids/phys2bids.py index 5161792bf..0d6a390a6 100644 --- a/phys2bids/phys2bids.py +++ b/phys2bids/phys2bids.py @@ -38,6 +38,7 @@ from phys2bids import _version, bids, utils, viz from phys2bids.cli.run import _get_parser from phys2bids.physio_obj import BlueprintOutput +from phys2bids.reporting.html_report import generate_report from phys2bids.slice4phys import slice4phys from . import __version__ @@ -145,6 +146,7 @@ def phys2bids( pad=9, ch_name=[], yml="", + make_report=False, debug=False, quiet=False, ): @@ -538,6 +540,10 @@ def phys2bids( ), ) + # Only generate report if specified by the user + if make_report: + generate_report(outdir, conversion_path, logname, phys_out[key]) + def _main(argv=None): options = _get_parser().parse_args(argv) diff --git a/phys2bids/reporting/__init__.py b/phys2bids/reporting/__init__.py new file mode 100644 index 000000000..adeeb7218 --- /dev/null +++ b/phys2bids/reporting/__init__.py @@ -0,0 +1 @@ +"""Visual reporting tools for inspecting phys2bids workflow outputs.""" diff --git a/phys2bids/reporting/assets/apple-icon-180x180.png b/phys2bids/reporting/assets/apple-icon-180x180.png new file mode 100644 index 000000000..4b6e2e4bf Binary files /dev/null and b/phys2bids/reporting/assets/apple-icon-180x180.png differ diff --git a/phys2bids/reporting/assets/main.css b/phys2bids/reporting/assets/main.css new file mode 100644 index 000000000..a5b9886bb --- /dev/null +++ b/phys2bids/reporting/assets/main.css @@ -0,0 +1,102 @@ +html, body { + margin: 0; + padding: 0; + font-family: 'Lato', sans-serif; + overflow-x: hidden; + overflow-y: scroll; +} +* { + box-sizing: border-box; + } + +.header { + background: linear-gradient(90deg, rgba(0,240,141,1) 0%, rgba(0,73,133,1) 100%); + height: 70px; + width: 100%; + position: fixed; + overflow: hidden; + margin: 0; + z-index: 100; +} + +.header a, span { + color: white; + text-decoration: none; + font-weight: 700; +} + +.header_logo { + display: inline-block; + float: left; +} + +.header_logo img{ + height: 50px; + top: 0; + left: 0; + padding-top: 15px; +} + +.header_links { + top: 0; + left: 0; + padding-top: 25px; + margin-left: 20px; + margin-right: 20px; + float: left; + display: inline-block; +} +.clear { + clear: both; +} + +.content { + margin-top: 100px; + display: flex; + width: 100%; +} + +.tree { + margin-left: 50px; + margin-right: 50px; + flex: 0.5; + min-width: 300px; + float: left; +} + +.tree_text { + margin-top: 10px; + margin-bottom: 70px; + width: 100%; +} + +.bk-root { + display: inline-block; + margin-top: 10px; + width: 100%; +} + +.bokeh_plots { + margin-left: 50px; + margin-right: 50px; + flex: 1; + min-width: 500px; + float: left; +} + +@media screen and (max-width: 600px) { + .content { + flex-wrap: wrap; + } + .tree { + flex-basis: 100%; + } + .bokeh_plots { + flex-basis: 100%; + } +} + +.main{ + margin-top: 100px; + margin-left: 100px; +} diff --git a/phys2bids/reporting/assets/phys2bids_negativeReadTheDocs_500x150.png b/phys2bids/reporting/assets/phys2bids_negativeReadTheDocs_500x150.png new file mode 100644 index 000000000..815be7e98 Binary files /dev/null and b/phys2bids/reporting/assets/phys2bids_negativeReadTheDocs_500x150.png differ diff --git a/phys2bids/reporting/html_report.py b/phys2bids/reporting/html_report.py new file mode 100644 index 000000000..3a04b10d1 --- /dev/null +++ b/phys2bids/reporting/html_report.py @@ -0,0 +1,247 @@ +"""Reporting functionality for phys2bids.""" + +import sys +from distutils.dir_util import copy_tree +from os.path import basename, join +from pathlib import Path +from string import Template + +from bokeh.embed import components +from bokeh.layouts import gridplot +from bokeh.plotting import ColumnDataSource, figure + +from phys2bids import _version + + +def _save_as_html(log_html_path, log_content, qc_html_path): + """ + Save an HTML report out to a file. + + Parameters + ---------- + log_html_path : str + Body for HTML report with embedded figures + log_content: str + String containing the logs generated by phys2bids + qc_html_path : str + Path to the quality check section of the report + + Returns + ------- + html: HTML code of the report + + Outcome + ------- + Saves the html file + """ + resource_path = Path(__file__).resolve().parent + head_template_name = "report_log_template.html" + head_template_path = resource_path.joinpath(head_template_name) + with open(str(head_template_path), "r") as head_file: + head_tpl = Template(head_file.read()) + + html = head_tpl.substitute( + version=_version.get_versions()["version"], + log_html_path=log_html_path, + log_content=log_content, + qc_html_path=qc_html_path, + ) + return html + + +def _update_fpage_template(tree_string, bokeh_id, bokeh_js, log_html_path, qc_html_path): + """ + Populate a report with content. + + Parameters + ---------- + tree_string: str + Tree of files in directory. + bokeh_id : str + HTML div created by bokeh.embed.components + bokeh_js : str + Javascript created by bokeh.embed.components + log_html_path : str + Path to the log section of the report + qc_html_path : str + Path to the quality check section of the report + + Returns + ------- + body : Body for HTML report with embedded figures + """ + resource_path = Path(__file__).resolve().parent + + body_template_name = "report_plots_template.html" + body_template_path = resource_path.joinpath(body_template_name) + with open(str(body_template_path), "r") as body_file: + body_tpl = Template(body_file.read()) + body = body_tpl.substitute( + tree=tree_string, + content=bokeh_id, + javascript=bokeh_js, + version=_version.get_versions()["version"], + log_html_path=log_html_path, + qc_html_path=qc_html_path, + ) + return body + + +def _generate_file_tree(out_dir): + """ + Populate a report with content. + + Parameters + ---------- + outdir : str + Path to the output directory + + Returns + ------- + tree_string: String with the tree of files in directory + """ + # prefix components: + space = " " + branch = "│ " + # pointers: + tee = "├── " + last = "└── " + + def tree(dir_path: Path, prefix: str = ""): + """Generate tree structure. + + Given a directory Path object + will yield a visual tree structure line by line + with each line prefixed by the same characters + + from https://stackoverflow.com/questions/9727673/list-directory-tree-structure-in-python + """ + contents = list(dir_path.iterdir()) + # contents each get pointers that are ├── with a final └── : + pointers = [tee] * (len(contents) - 1) + [last] + for pointer, path in zip(pointers, contents): + yield prefix + pointer + path.name + if path.is_dir(): # extend the prefix and recurse: + extension = branch if pointer == tee else space + # i.e. space because last, └── , above so no more | + yield from tree(path, prefix=prefix + extension) + + tree_string = "" + for line in tree(Path(out_dir)): + tree_string += line + "
" + return tree_string + + +def _generate_bokeh_plots(phys_in, figsize=(250, 500)): + """ + Plot all the channels for visualizations as linked line plots for dynamic report. + + Parameters + ---------- + phys_in: BlueprintInput object + Object returned by BlueprintInput class + figsize: tuple + Size of the figure expressed as (size_x, size_y), + Default is 250x750px + + Outcome + ------- + Creates new plot with path specified in outfile. + + See Also + -------- + https://phys2bids.readthedocs.io/en/latest/howto.html + """ + colors = ["#ff7a3c", "#008eba", "#ff96d3", "#3c376b", "#ffd439"] + + time = phys_in.timeseries.T[0] # assumes first phys_in.timeseries is time + ch_num = len(phys_in.ch_name) + if ch_num > len(colors): + colors *= 2 + + if phys_in.freq > 100: + downsample = int(phys_in.freq / 100) + else: + downsample = None + + plot_list = [] + for row, timeser in enumerate(phys_in.timeseries.T[1:]): + # build a data source for each plot, with only the data + index (time) + # for the purpose of reporting, data is downsampled 10x + # doesn't make much of a difference to the naked eye, fine for reports + source = ColumnDataSource(data=dict(x=time[::downsample], y=timeser[::downsample])) + + i = row + 1 + + tools = ["wheel_zoom,pan,reset"] + q = figure( + plot_height=figsize[0], + plot_width=figsize[1], + tools=tools, + title=f" Channel {i}: {phys_in.ch_name[i]}", + sizing_mode="stretch_both", + ) + q.line("x", "y", color=colors[i - 1], alpha=0.9, source=source) + q.xaxis.axis_label = "Time (s)" + # hovertool commented for posterity because I (KB) will be triumphant + # eventually + # q.add_tools(HoverTool(tooltips=[ + # (phys_in.ch_name[i], '@y{0.000} ' + phys_in.units[i]), + # ('HELP', '100 :D') + # ], mode='vline')) + plot_list.append([q]) + p = gridplot( + plot_list, toolbar_location="right", plot_height=250, plot_width=750, merge_tools=True + ) + script, div = components(p) + return script, div + + +def generate_report(out_dir, conversion_path, log_path, phys_in): + """ + Plot all the channels for visualizations as linked line plots for dynamic report. + + Parameters + ---------- + out_dir : str + File path to a completed phys2bids output directory + log_path: path + Path to the logged output of phys2bids + phys_in: BlueprintInput object + Object returned by BlueprintInput class + + Outcome + ------- + Creates new plot with path specified in outfile. + + See Also + -------- + https://phys2bids.readthedocs.io/en/latest/howto.html + """ + # Copy assets into output folder + pkgdir = sys.modules["phys2bids"].__path__[0] + assets_path = join(pkgdir, "reporting", "assets") + copy_tree(assets_path, join(conversion_path, "assets")) + + # Read log + with open(log_path, "r") as f: + log_content = f.read() + + log_content = log_content.replace("\n", "
") + log_html_path = join(conversion_path, basename(phys_in.filename) + ".html") + qc_html_filename = ( + "_".join(basename(phys_in.filename).split("_")[:-1]) + "_desc-log_physio.html" + ) + qc_html_path = join(conversion_path, qc_html_filename) + html = _save_as_html(log_html_path, log_content, qc_html_path) + + with open(log_html_path, "wb") as f: + f.write(html.encode("utf-8")) + + # Read in output directory structure & create tree + tree_string = _generate_file_tree(out_dir) + bokeh_js, bokeh_div = _generate_bokeh_plots(phys_in, figsize=(250, 750)) + html = _update_fpage_template(tree_string, bokeh_div, bokeh_js, log_html_path, qc_html_path) + + with open(qc_html_path, "wb") as f: + f.write(html.encode("utf-8")) diff --git a/phys2bids/reporting/report_log_template.html b/phys2bids/reporting/report_log_template.html new file mode 100644 index 000000000..fb1cfb32f --- /dev/null +++ b/phys2bids/reporting/report_log_template.html @@ -0,0 +1,33 @@ + + + + phys2bids report + + + + + +
+ +
+ phys2bids $version +
+
+ Quality Check +
+
+ Log +
+
+ GitHub +
+
+
+

$log_content

+
+ + diff --git a/phys2bids/reporting/report_plots_template.html b/phys2bids/reporting/report_plots_template.html new file mode 100644 index 000000000..fef38570e --- /dev/null +++ b/phys2bids/reporting/report_plots_template.html @@ -0,0 +1,50 @@ + + + + + phys2bids report + + + + + + + + +
+ +
+ phys2bids $version +
+
+ Quality Check +
+
+ Log +
+
+ GitHub +
+
+
+
+

Channel Plots

+

Hold on... It might take a few seconds to load the plots depending on how big your data is.

+ $content +
+
+

phys2BIDS Output Directory

+

$tree

+
+
+ + + +$javascript diff --git a/phys2bids/tests/test_integration.py b/phys2bids/tests/test_integration.py index 88ae22680..04dd3121b 100644 --- a/phys2bids/tests/test_integration.py +++ b/phys2bids/tests/test_integration.py @@ -119,7 +119,7 @@ def test_integration_heuristic(skip_integration, multifreq_lab_file): f"phys2bids -in {test_full_path} ", f"-chtrig {test_chtrig} -outdir {test_outdir} ", f"-tr {test_tr} -ntp {test_ntp} -thr {test_thr} ", - f"-sub 006 -ses 01 -heur {test_heur}", + f"-sub 006 -ses 01 -heur {test_heur} -report", ) command_str = "".join(command_str) subprocess.run(command_str, shell=True, check=True) @@ -128,7 +128,7 @@ def test_integration_heuristic(skip_integration, multifreq_lab_file): assert isfile(join(conversion_path, "call.sh")) # Read logger file - logger_file = glob.glob(join(conversion_path, "*phys2bids*"))[0] + logger_file = glob.glob(join(conversion_path, "*phys2bids*.tsv"))[0] with open(logger_file) as logger_info: logger_info = logger_info.readlines() diff --git a/setup.cfg b/setup.cfg index 7d11eb98a..7f9e86fd1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -48,6 +48,8 @@ doc = sphinx >=2.0 sphinx-argparse sphinx_rtd_theme +reports = + bokeh ==2.4.3 style = flake8 >=4.0 black @@ -64,11 +66,13 @@ test = coverage requests %(interfaces)s + %(reports)s %(style)s all = %(doc)s %(duecredit)s %(interfaces)s + %(reports)s %(style)s %(test)s