diff --git a/.gitignore b/.gitignore index 45062fbc63610d652c5ba4414abb7bd3b28b6ad1..c268e75455d68123f9067f2836060e3ecd574ab3 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,4 @@ __pycache__/ *.egg-info/ .eggs/ /doc/_generated - +/doc/tasks diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bff79bbd377207f3024bb96b332a835083899c8f..8e7e72cfb993220d5b1329c573971132998cad57 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,6 +4,11 @@ include: build_doc: extends: .build_doc + before_script: + - python -m pip install . + - python -m pip list + - python scripts/generate.py --install + - !reference [.build_doc, before_script] only: refs: - main diff --git a/doc/index.md b/doc/index.md index 555f3c40a348bc130aab6907a64e3c68d6c998d3..89e06e066b1c87e24e0b712263087206016090ff 100644 --- a/doc/index.md +++ b/doc/index.md @@ -2,120 +2,12 @@ ## Beamlines using Ewoks -:::{admonition} 25 beamlines use Ewoks to process their data! -:class: info -See the list below regrouped by technique. -::: - -::::{grid} 3 -:gutter: 3 - -:::{grid-item-card} Tomography -:link: tasks/tomography.html - -â‡ï¸ **ID19, BM05, BM18, ID11, ID16B, ID17** - -ðŸ–¼ï¸ **Qt** - -âš™ï¸ **[tomwer](https://gitlab.esrf.fr/tomotools/tomwer)** -::: - -:::{grid-item-card} MX beamline automation -â‡ï¸ **ID23-1, ID23-2, ID30A-1, ID30A-3, ID30B** - -ðŸ–¼ï¸ **Web** - -âš™ï¸ **[BES](https://gitlab.esrf.fr/svensson/bes)** -::: - -:::{grid-item-card} Dark-field microscopy -â‡ï¸ **ID06, ID11** - -ðŸ–¼ï¸ **Qt** - -âš™ï¸ **[darfix](https://gitlab.esrf.fr/XRD/darfix)** -::: - -:::{grid-item-card} BioSAXS -â‡ï¸ **BM29** -::: - -:::{grid-item-card} SAXS/WAXS data reduction (pyfai) -â‡ï¸ **ID31, BM02, ID11, STREAMLINE** - -ðŸ–¼ï¸ **Qt** - -âš™ï¸ **[ewoksxrpd](https://gitlab.esrf.fr/workflow/ewoksapps/ewoksxrpd)** -::: - -:::{grid-item-card} Powder Diffraction (high-resolution) -â‡ï¸ **ID22** - -âš™ï¸ **[ewoksid22](https://gitlab.esrf.fr/workflow/ewoksapps/ewoksid22)** -::: - -:::{grid-item-card} Diffraction -â‡ï¸ **ID11** - -ðŸ–¼ï¸ **Qt** - -âš™ï¸ **[ewoksid11](https://gitlab.esrf.fr/workflow/ewoksapps/ewoksid11)** -::: - -:::{grid-item-card} Inelastic Scattering (ixstools) -â‡ï¸ **ID20** - -ðŸ–¼ï¸ **Qt** - -âš™ï¸ **[ewoksixs](https://gitlab.esrf.fr/workflow/ewoksapps/ewoksixs)** -::: - -:::{grid-item-card} Spectroscopy (larch, pymca) -â‡ï¸ **BM23, ID24** - -ðŸ–¼ï¸ **Qt** - -âš™ï¸ **[est](https://gitlab.esrf.fr/workflow/ewoksapps/est)** -::: - -:::{grid-item-card} Spectroscopy (custom tools) -â‡ï¸ **ID26** -::: - -:::{grid-item-card} Fluorescence (pymca, image reg) -**ID21, ID16A, ID16B** - -ðŸ–¼ï¸ **Qt** - -**[ewoksfluo](https://gitlab.esrf.fr/workflow/ewoksapps/ewoksfluo)**; **[ewoksndreg](https://gitlab.esrf.fr/workflow/ewoksapps/ewoksndreg)** -::: - -:::{grid-item-card} ID10 -_Under construction ðŸ—ï¸_ -::: - -:::{grid-item-card} ID09 -_Under construction ðŸ—ï¸_ -::: - -:::{grid-item-card} BM16 -_Under construction ðŸ—ï¸_ -::: - -:::: - -## Software - -Workflow Task Libraries +```{include} tasks/overview.md +``` -- [Ewoks core projects](https://gitlab.esrf.fr/workflow/ewoks) -- [Ewoks task libraries](https://gitlab.esrf.fr/workflow/ewoksapps) -- External Ewoks task libraries: - - [Tomography workflows](https://gitlab.esrf.fr/tomotools/tomwer) - - [Darkfield microscopy workflows](https://gitlab.esrf.fr/XRD/darfix) - - [MX automation workflows](https://gitlab.esrf.fr/svensson/bes) +## Usefull libraries -Data and resource management +Python libraries that are related or used by ewoks - [Bliss data access](https://gitlab.esrf.fr/bliss/blissdata) - [Online Data Processing in Bliss](https://gitlab.esrf.fr/bliss/blissoda) diff --git a/doc/tasks/index.md b/doc/tasks/index.md deleted file mode 100644 index c1191ddd1d42bbea1cfc0312f07769b2686213a8..0000000000000000000000000000000000000000 --- a/doc/tasks/index.md +++ /dev/null @@ -1,5 +0,0 @@ -# Tasks per category - -:::{toctree} -tomography -::: diff --git a/doc/tasks/tomography.md b/doc/tasks/tomography.md deleted file mode 100644 index a06229f6f599a57d5084d014159936140164c057..0000000000000000000000000000000000000000 --- a/doc/tasks/tomography.md +++ /dev/null @@ -1,12 +0,0 @@ -# Tomography - -:::{admonition} TL;DR -:class: hint - -â‡ï¸ **ID19, BM05, BM18, ID11, ID16B, ID17** - -ðŸ–¼ï¸ **Qt** - -âš™ï¸ **[tomwer](https://gitlab.esrf.fr/tomotools/tomwer)** - -::: diff --git a/pyproject.toml b/pyproject.toml index 614dba60c52ab4ebdc4669300f4692b7f20e6355..7b49e7c96d502bb62cc928a1e607f9cfe77fdc7f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,8 @@ dependencies = [ 'sphinx', 'sphinx-autobuild', 'sphinx-book-theme', - 'sphinx-design' + 'sphinx-design', + 'tabulate', + 'ewokscore' ] readme = 'README.md' diff --git a/scripts/generate.py b/scripts/generate.py new file mode 100644 index 0000000000000000000000000000000000000000..84a71e0e2b584420bd661222123672c32b9b27cb --- /dev/null +++ b/scripts/generate.py @@ -0,0 +1,186 @@ +import os +import re +import sys +import argparse +import subprocess +from tabulate import tabulate +from importlib import metadata +from typing import Tuple, Dict, List + +from ewokscore.task_discovery import discover_all_tasks + + +_PROJECT_TO_CATEGORY = { + "ewoksxrpd": "SAXS/WAXS", + "ewoksfluo": "Fluorescence", + "ewoksndreg": "Imaging", + "ewokscore": "Demo", + "tomwer": "Tomography", + "darfix": "Dark-field microscopy", + "bes": "MX beamline automation", + "ewoksixs": "Inelastic Scattering", + "ewoksid22": "Special diffraction", + "ewoksid11": "Special diffraction", + "est": "Spectroscopy", + "ewoksdata": "Data Access", +} + +_PROJECT_WITH_QT = [ + "ewoksxrpd", + "ewoksfluo", + "ewoksndreg", + "ewokscore", + "tomwer", + "darfix", + "est", +] + + +_CATEGORY_TO_BEAMLINES = { + "Tomography": ["ID19", "BM05", "BM18", "ID11", "ID16B", "ID17"], + "SAXS/WAXS": ["ID31", "BM02", "ID11", "STREAMLINE"], + "Spectroscopy": ["BM23", "ID24"], + "Fluorescence": ["ID21", "ID16A", "ID16B"], + "Dark-field microscopy": ["ID06", "ID11"], + "Imaging": ["ID21", "ID16b"], + "MX beamline automation": ["ID23-1", "ID23-2", "ID30A-1", "ID30A-3", "ID30B"], + "BioSAXS": ["BM29"], + "Special diffraction": ["ID22", "ID11"], + "Special Spectroscopy": ["ID26"], + "Data Access": [], + "Demo": [], + "Development": ["ID10", "ID09", "BM16"], +} + + +def install(package: str) -> None: + if package in ("ewoksfluo", "ewoksixs", "ewoksid11"): + return + subprocess.check_call([sys.executable, "-m", "pip", "install", package]) + + +def create_tables() -> Tuple[Dict[str, List[str]], List[str]]: + tables = {category: list() for category in _CATEGORY_TO_BEAMLINES} + + for task in discover_all_tasks(): + project = task["category"] + category = _PROJECT_TO_CATEGORY.get(project, "Miscellaneous") + description = task["description"] + if description: + description = description.split("\n")[0] + else: + description = "" + table = tables.setdefault(category, list()) + identifier = task["task_identifier"] + name = identifier.split(".")[-1] + table.append([name, description, project, identifier]) + + headers = ["Name", "Description", "Project", "Identifier"] + return headers, tables + + +def save_tables(headers: List[str], tables: Dict[str, List[str]], docdir: str) -> None: + dirname = os.path.join(docdir, "tasks") + os.makedirs(dirname, exist_ok=True) + + categories = list() + for category, table in tables.items(): + projects = {row[2] for row in table} + beamlines = _CATEGORY_TO_BEAMLINES.get(category, list()) + lines = [f"# {category}", "", ":::{admonition} TL;DR", ":class: hint", ""] + + if beamlines: + beamlines = sorted(set(beamlines)) + lines.extend([f"â‡ï¸ **{', '.join(beamlines)}**", ""]) + if any(project in _PROJECT_WITH_QT for project in projects): + lines.extend(["ðŸ–¼ï¸ **Qt**", ""]) + for project in projects: + url = metadata.metadata(project)["url"] + if url: + lines.extend([f"âš™ï¸ **[{project}]({url})**", ""]) + else: + lines.extend([f"âš™ï¸ **{project}**", ""]) + if not projects: + lines.extend(["_Under construction ðŸ—ï¸_", ""]) + + lines.extend([":::", ""]) + + category = re.sub(r"[^-0-9a-z]", "_", category.lower()) + categories.append(category) + with open(os.path.join(dirname, category + ".md"), "w") as f: + f.write("\n".join(lines)) + if table: + f.write("\n") + f.write(tabulate(table, headers, tablefmt="github")) + + with open(os.path.join(dirname, "index.md"), "w") as f: + f.write("# Tasks per category\n") + f.write("\n") + f.write(":::{toctree}\n") + for category in categories: + f.write(f"{category}\n") + f.write(":::\n") + + +def save_overview(tables: Dict[str, List[str]], docdir): + dirname = os.path.join(docdir, "tasks") + os.makedirs(dirname, exist_ok=True) + + lines = list() + all_beamlines = set() + for category, table in tables.items(): + lines.append(f":::{{grid-item-card}} {category}") + + projects = {row[2] for row in table} + beamlines = _CATEGORY_TO_BEAMLINES.get(category, list()) + + category = re.sub(r"[^-0-9a-z]", "_", category.lower()) + lines.extend([f":link: tasks/{category}.html", ""]) + + if beamlines: + beamlines = set(beamlines) + all_beamlines |= beamlines + lines.extend([f"â‡ï¸ **{', '.join(sorted(beamlines))}**", ""]) + + if not projects: + lines.extend(["_Under construction ðŸ—ï¸_", ""]) + + lines.extend([":::", ""]) + + with open(os.path.join(dirname, "overview.md"), "w") as f: + nbeamlines = f"{len(all_beamlines)} beamlines" + f.write(f":::{{admonition}} {nbeamlines} use Ewoks to process their data!\n") + f.write(":class: info\n") + f.write("See the list below regrouped by technique.\n") + f.write(":::\n") + f.write("\n") + f.write("::::{grid} 3\n") + f.write(":gutter: 3\n") + f.write("\n") + f.write("\n".join(lines)) + f.write("::::\n") + + +def main(argv=None): + parser = argparse.ArgumentParser(description="Generate ewoks task docs") + parser.add_argument( + "--install", + action="store_true", + help="Install libraries", + ) + + if argv is None: + argv = sys.argv + args = parser.parse_args(argv[1:]) + + if args.install: + for project in _PROJECT_TO_CATEGORY: + install(project) + headers, tables = create_tables() + docdir = os.path.join(os.path.dirname(__file__), "..", "doc") + save_tables(headers, tables, docdir) + save_overview(tables, docdir) + + +if __name__ == "__main__": + main() diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000000000000000000000000000000000000..a06f8cdca039ba65b57e8d03d77d388610597022 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,9 @@ +# setup.cfg + +[flake8] +# Recommend matching the black line length (default 88), +# rather than using the flake8 default of 79: +max-line-length = 88 +extend-ignore = + # See https://github.com/PyCQA/pycodestyle/issues/373 + E203, \ No newline at end of file