diff --git a/.github/actions/workflowhub/action.yml b/.github/actions/workflowhub/action.yml new file mode 100644 index 00000000..d731d4c4 --- /dev/null +++ b/.github/actions/workflowhub/action.yml @@ -0,0 +1,18 @@ +name: Update WorkflowHub data +description: Generate validated Jekyll data from a configured WorkflowHub collection + +runs: + using: composite + steps: + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.12" + + - name: Install dependencies + shell: bash + run: python -m pip install "requests>=2.32,<3" "PyYAML>=6,<7" + + - name: Generate WorkflowHub data + shell: bash + run: python "$GITHUB_ACTION_PATH/workflowhub_finder.py" diff --git a/.github/actions/workflowhub/workflowhub_finder.py b/.github/actions/workflowhub/workflowhub_finder.py new file mode 100644 index 00000000..fad1b0f8 --- /dev/null +++ b/.github/actions/workflowhub/workflowhub_finder.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +import os +import tempfile +from pathlib import Path + +import requests +import yaml + + +DEFAULT_BASE_URL = "https://workflowhub.eu" +OUTPUT_PATH = Path("_data/data_workflows.yaml") +REQUEST_TIMEOUT = 30.0 +REQUIRED_FIELDS = ("title", "url") + + +class WorkflowHubClient: + def __init__(self, base_url, collection_id, timeout=REQUEST_TIMEOUT): + self.base_url = base_url.rstrip("/") + self.collection_id = collection_id + self.timeout = timeout + + def _get_json(self, path): + response = requests.get(f"{self.base_url}/{path}", timeout=self.timeout) + response.raise_for_status() + return response.json() + + def _workflow_ids(self): + items = self._get_json( + f"collections/{self.collection_id}/items.json" + )["data"] + return [ + item["relationships"]["asset"]["data"]["id"] for item in items + ] + + def _projects(self): + projects = self._get_json("projects.json")["data"] + return { + project["id"]: project["attributes"]["title"] for project in projects + } + + @staticmethod + def _render_workflow(metadata, project_titles): + data = metadata["data"] + attributes = data["attributes"] + latest_version = attributes["latest_version"] + remote = next( + ( + version.get("remote", "") + for version in attributes.get("versions", []) + if version.get("version") == latest_version + ), + "", + ) + project_ids = ( + data.get("relationships", {}).get("projects", {}).get("data", []) + ) + projects = list( + dict.fromkeys( + project_titles[item["id"]] + for item in project_ids + if item["id"] in project_titles + ) + ) + workflow_class = attributes.get("workflow_class") or {} + + return { + "title": attributes.get("title", "").replace("_", " "), + "class": workflow_class.get("key", ""), + "url": data.get("links", {}).get("self", ""), + "remote_link": remote, + "edam_top": [ + annotation["label"] + for annotation in attributes.get("topic_annotations", []) + ] or "", + "edam_ops": [ + annotation["label"] + for annotation in attributes.get("operation_annotations", []) + ] or "", + "license": attributes.get("license") or "", + "updated_at": (attributes.get("updated_at") or "").split("T")[0], + "doi": attributes.get("doi") or "", + "projects": projects or "", + "tools": attributes.get("tools") or "", + } + + def workflows(self): + project_titles = self._projects() + workflows = [ + self._render_workflow( + self._get_json(f"workflows/{workflow_id}.json"), project_titles + ) + for workflow_id in sorted(set(self._workflow_ids()), key=str) + ] + return sorted( + workflows, + key=lambda workflow: (workflow["title"].casefold(), workflow["url"]), + ) + + +def load_workflows(path): + try: + data = yaml.safe_load(path.read_text(encoding="utf-8")) + except (OSError, yaml.YAMLError) as error: + raise ValueError(f"Could not read valid YAML from {path}: {error}") from error + if not isinstance(data, list): + raise ValueError(f"{path} must contain a top-level list") + return data + + +def validate_workflows(workflows): + urls = set() + for index, workflow in enumerate(workflows, start=1): + if not isinstance(workflow, dict): + raise ValueError(f"Workflow {index} must be a mapping") + missing = [field for field in REQUIRED_FIELDS if not workflow.get(field)] + if missing: + raise ValueError( + f"Workflow {index} is missing required values: {', '.join(missing)}" + ) + if workflow["url"] in urls: + raise ValueError(f"Duplicate workflow URL: {workflow['url']}") + urls.add(workflow["url"]) + + +def serialise_workflows(workflows): + return yaml.safe_dump( + workflows, + default_flow_style=False, + sort_keys=True, + allow_unicode=True, + ) + + +def update_workflow_file(workflows, output=OUTPUT_PATH): + baseline = load_workflows(output) if output.exists() else None + validate_workflows(workflows) + if baseline and not workflows: + raise ValueError("Refusing to replace a non-empty dataset with an empty result") + rendered = serialise_workflows(workflows) + + if output.exists() and output.read_text(encoding="utf-8") == rendered: + return False + + output.parent.mkdir(parents=True, exist_ok=True) + temporary_path = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + dir=output.parent, + prefix=f".{output.name}.", + suffix=".tmp", + delete=False, + ) as temporary_file: + temporary_file.write(rendered) + temporary_path = Path(temporary_file.name) + os.replace(temporary_path, output) + finally: + if temporary_path and temporary_path.exists(): + temporary_path.unlink() + return True + + +def load_site_config(path): + try: + config = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + except (OSError, yaml.YAMLError) as error: + raise ValueError(f"Could not read site configuration from {path}: {error}") from error + + integration = config.get("workflowhub") + if not isinstance(integration, dict) or not integration.get("enabled", False): + return None + if integration.get("collection_id") in (None, ""): + raise ValueError("workflowhub.collection_id is required when enabled") + return { + "base_url": integration.get("base_url") or DEFAULT_BASE_URL, + "collection_id": int(integration["collection_id"]), + } + + +def main(): + settings = load_site_config(Path("_config.yml")) + if settings is None: + print("WorkflowHub integration is disabled; nothing to generate") + return + + client = WorkflowHubClient(**settings) + workflows = client.workflows() + changed = update_workflow_file(workflows) + status = "Updated" if changed else "No changes to" + print(f"{status} {OUTPUT_PATH} ({len(workflows)} workflows)") + + +if __name__ == "__main__": + try: + main() + except (ValueError, requests.RequestException) as error: + raise SystemExit(f"WorkflowHub update failed: {error}") from error diff --git a/.github/workflows/workflowhub.yml b/.github/workflows/workflowhub.yml new file mode 100644 index 00000000..0db239c0 --- /dev/null +++ b/.github/workflows/workflowhub.yml @@ -0,0 +1,48 @@ +name: Update WorkflowHub data + +on: + workflow_call: + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +concurrency: + group: update-workflowhub-data + cancel-in-progress: false + +jobs: + update: + runs-on: ubuntu-24.04 + steps: + - name: Check out website + uses: actions/checkout@v7 + with: + ref: ${{ github.event.repository.default_branch }} + + - name: Check out WorkflowHub integration + uses: actions/checkout@v7 + with: + repository: ${{ job.workflow_repository }} + ref: ${{ job.workflow_sha }} + path: .workflowhub-theme + + - name: Update WorkflowHub data + uses: ./.workflowhub-theme/.github/actions/workflowhub + + - name: Remove integration checkout + shell: bash + run: rm -rf -- .workflowhub-theme + + - name: Create or update pull request + uses: peter-evans/create-pull-request@v8 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: Update WorkflowHub workflow data + title: Update WorkflowHub workflow data + body: Automated refresh of validated WorkflowHub collection data. + branch: automation/update-workflowhub-data + base: ${{ github.event.repository.default_branch }} + add-paths: _data/data_workflows.yaml + delete-branch: true diff --git a/.gitignore b/.gitignore index eb417726..e0cc5eb3 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,5 @@ _site/ vendor/ .vscode/ Gemfile.lock -.ruby-version \ No newline at end of file +.ruby-version +**/__pycache__/ diff --git a/_config.yml b/_config.yml index 7071ba12..bc44d12c 100644 --- a/_config.yml +++ b/_config.yml @@ -10,6 +10,10 @@ title: ELIXIR Toolkit Theme description: "Flexible Jekyll theme using bootstrap 5 as CSS framework." # Metadata description of the website +workflowhub: + enabled: true + collection_id: 40 + dsw_deep_link_prefix: https://researchers.ds-wizard.org/knowledge-models/dsw:root:latest/preview?questionUuid= # prefix for DSW deep links to a certain question diff --git a/_data/data_workflows.yaml b/_data/data_workflows.yaml new file mode 100644 index 00000000..b77b3f3b --- /dev/null +++ b/_data/data_workflows.yaml @@ -0,0 +1,31 @@ +- class: nextflow + doi: 10.48546/workflowhub.workflow.2318.1 + edam_ops: + - Protein interaction prediction + - Protein quaternary structure prediction + - Protein structure analysis + - Protein structure prediction + edam_top: + - Protein folds and structural domains + - Protein structure analysis + - Structure prediction + license: MIT + projects: + - Lucet Laboratory (WEHI) + remote_link: '' + title: nf-boltz-2 + tools: '' + updated_at: '2026-09-28' + url: /workflows/2318?version=1 +- class: nextflow + doi: '' + edam_ops: '' + edam_top: '' + license: MIT + projects: + - nf-core + remote_link: https://github.com/nf-core/proteinfold.git + title: nf-core/proteinfold + tools: '' + updated_at: '2026-05-13' + url: /workflows/1011?version=3 diff --git a/_data/sidebars/main.yml b/_data/sidebars/main.yml index 92f48727..4a5fa823 100644 --- a/_data/sidebars/main.yml +++ b/_data/sidebars/main.yml @@ -33,6 +33,8 @@ subitems: url: /resource_table - title: Supported by url: /supported_by + - title: WorkflowHub workflows table + url: /workflowhub_integration - title: Versions and upgrades icon: icon-refresh-cw subitems: @@ -70,6 +72,8 @@ subitems: url: /general_page_7 - title: All training url: /all_trainings + - title: Workflows + url: /workflows - title: List of page IDs url: /list_page_ids - title: Page Not Found diff --git a/_includes/workflow-table.html b/_includes/workflow-table.html new file mode 100644 index 00000000..5d998a84 --- /dev/null +++ b/_includes/workflow-table.html @@ -0,0 +1,67 @@ +{%- assign workflows = site.data.data_workflows -%} +{%- if workflows -%} + {%- assign workflows = workflows | sort_natural: "title" -%} +{%- endif -%} +{%- if workflows and workflows.size > 0 -%} +Skip workflow table +
+ + + + + + + + + + + + + + {%- for workflow in workflows -%} + {%- assign workflow_url = workflow.url -%} + {%- assign first_character = workflow_url | slice: 0, 1 -%} + {%- if first_character == "/" -%} + {%- assign workflowhub_base_url = site.workflowhub.base_url | default: "https://workflowhub.eu" -%} + {%- assign workflow_url = workflow_url | prepend: workflowhub_base_url -%} + {%- endif -%} + + + + + + + + + + {%- endfor -%} + +
WorkflowTypeTopicsOperationsLicenseUpdatedProjects
+ {{ workflow.title | escape }} + {%- if workflow.remote_link != blank or workflow.doi != blank -%} +
+ {%- if workflow.remote_link != blank %}Source{% endif -%} + {%- if workflow.remote_link != blank and workflow.doi != blank %}{% endif -%} + {%- if workflow.doi != blank %}DOI{% endif -%} +
+ {%- endif -%} + 		{{ workflow.class | escape }} + {%- for topic in workflow.edam_top -%} + {{ topic | escape }}{% unless forloop.last %} {% endunless %} + {%- endfor -%} + + {%- for operation in workflow.edam_ops -%} + {{ operation | escape }}{% unless forloop.last %} {% endunless %} + {%- endfor -%} + {{ workflow.license | escape }}{{ workflow.updated_at | escape }} + {%- for project in workflow.projects -%} + {{ project | escape }}{% unless forloop.last %}
{% endunless %} + {%- endfor -%} +
+
+
+{%- else -%} +
+ No WorkflowHub records are available. Enable the WorkflowHub integration and run its update workflow to populate this table. +
+{%- endif -%} diff --git a/pages/documentation/configuring_theme.md b/pages/documentation/configuring_theme.md index db7177e6..5fce1c76 100644 --- a/pages/documentation/configuring_theme.md +++ b/pages/documentation/configuring_theme.md @@ -82,6 +82,11 @@ theme_variables: fonts: - url to a font breadcrumb: false + +workflowhub: + enabled: false + collection_id: + base_url: https://workflowhub.eu ``` More detailed information about these settings can be found here: @@ -123,3 +128,9 @@ More detailed information about these settings can be found here: * **`theme_color`**: This is the primary theme color which is used in the metadata of the website. Please use the hex color without the hashtag as value. * **`fonts`**: List URLs to Google Fonts or other custom fonts for your website. * **`breadcrumb`**: Adds a breadcrumb above the title of every page. The breadcrumb will always represent the URL pattern and works best in combination with [permalink style](https://jekyllrb.com/docs/permalinks/#built-in-formats) *pretty*. Default: *false* +* **workflowhub**: Top-level settings for WorkflowHub links and optional collection-based data generation. Do not nest this block under `theme_variables`. + * **`enabled`**: Enable collection-based WorkflowHub data generation. This is not required when maintaining the data file manually. Default: *false*. + * **`collection_id`**: ID of the WorkflowHub collection to retrieve. Required when enabled. + * **`base_url`**: WorkflowHub instance to query. Default: `https://workflowhub.eu`. + +See the [WorkflowHub workflows table documentation](workflowhub_integration) for manual maintenance, table rendering and optional GitHub Actions automation. diff --git a/pages/documentation/workflowhub_integration.md b/pages/documentation/workflowhub_integration.md new file mode 100644 index 00000000..946a7c4d --- /dev/null +++ b/pages/documentation/workflowhub_integration.md @@ -0,0 +1,158 @@ +--- +title: WorkflowHub workflows table +datatable: true +--- + +The theme can display workflows in a searchable table. Workflow metadata is stored in `_data/data_workflows.yaml`, following the same data-file pattern as the [tools and resources table](resource_table). + +The data file can be maintained manually. Sites that use a public WorkflowHub collection can optionally use the supplied GitHub Actions workflow to keep it up to date. + +## Way of working + +The `{% raw %}{% include workflow-table.html %}{% endraw %}` include reads the committed `_data/data_workflows.yaml` file. The file can be edited manually or generated from a WorkflowHub collection. Jekyll does not contact WorkflowHub when it builds the site. + +## Prerequisites + +* Use a recent version of the ELIXIR Toolkit Theme. +* Create `_data/data_workflows.yaml` manually or generate it from a WorkflowHub collection. +* Add `datatable: true` to the front matter of pages that render the table. + +## The main YAML file + +`_data/data_workflows.yaml` contains a top-level list of workflow records. Only `title` and `url` are required. The supported fields are: + +* **`title`**: Display name of the workflow. +* **`url`**: Absolute WorkflowHub URL, or a path relative to the configured `workflowhub.base_url`. +* **`class`**: Workflow type, such as `nextflow`, `galaxy` or `snakemake`. +* **`edam_top`**: List of EDAM topics. +* **`edam_ops`**: List of EDAM operations. +* **`license`**: Workflow licence identifier. +* **`updated_at`**: Last update date in `YYYY-MM-DD` format. +* **`projects`**: List of contributing projects or teams. +* **`doi`**: DOI without the `https://doi.org/` prefix. +* **`remote_link`**: Link to the source repository. +* **`tools`**: Tool metadata supplied by WorkflowHub. This is retained in the data file but is not currently displayed by the table. + +For example: + +```yml +- title: Example workflow + url: https://workflowhub.eu/workflows/123?version=1 + class: nextflow + edam_top: + - Sequence analysis + edam_ops: + - Sequence alignment + license: MIT + updated_at: '2026-01-31' + projects: + - Example project + doi: 10.48546/workflowhub.workflow.123.1 + remote_link: https://github.com/example/example-workflow + tools: '' +``` + +## Maintaining workflows manually + +1. Create `_data/data_workflows.yaml` if it does not already exist. +2. Add or update records using the fields above. Use two spaces for YAML indentation. +3. Check that every record has a unique, non-empty `url` and a non-empty `title`. +4. Preview the site locally and commit the updated data file. + +Manual data maintenance does not require `workflowhub.enabled` or a `collection_id`. When a record uses a relative `url`, the table resolves it against `workflowhub.base_url`, which defaults to `https://workflowhub.eu`. + +## Listing workflows + +Add `datatable: true` to the page front matter: + +```yml +--- +title: Workflows +datatable: true +--- +``` + +Then add the table include where it should appear: + +```liquid +{% raw %}{% include workflow-table.html %}{% endraw %} +``` + +This renders as: + +{% include workflow-table.html %} + +See the [workflows example page](workflows) for the rendered result. + +## Generating the data manually + +Instead of maintaining individual records, the supplied generator can retrieve all workflows in a public WorkflowHub collection. This requires a source checkout of the theme because maintenance scripts are not installed by the gem or remote theme. + +First configure the collection in the website's `_config.yml`: + +```yml +workflowhub: + enabled: true + collection_id: 40 + # base_url: https://workflowhub.eu +``` + +Replace `40` with the collection ID for the site. `base_url` is optional and defaults to `https://workflowhub.eu`. + +Create a Python environment and install the generator dependencies: + +```sh +python3 -m venv .venv +. .venv/bin/activate +python -m pip install "requests>=2.32,<3" "PyYAML>=6,<7" +``` + +From the root of the website repository, run the script from the theme checkout: + +```sh +python /path/to/elixir-toolkit-theme/.github/actions/workflowhub/workflowhub_finder.py +``` + +The command writes `_data/data_workflows.yaml`. Review the resulting diff, preview the site and commit the data file. Re-run the command whenever the collection needs to be refreshed. + +## Optional automated updates + +The GitHub Actions workflow automates the same generation process and opens a pull request when the collection changes. It is optional; the table and local generator do not depend on it. + +When it runs, the workflow: + +1. Checks out the website's default branch. +2. Loads the generator from the same theme commit as the workflow. +3. Reads the website's WorkflowHub settings and generates `_data/data_workflows.yaml`. +4. Opens or updates a pull request if the data changed. It does not write directly to the default branch. + +The workflow uses the repository's `GITHUB_TOKEN`; a separate WorkflowHub or personal access token is not required. + +### Forks + +Forks inherit `.github/workflows/workflowhub.yml` and the generator, so they do not need a caller workflow. Enable GitHub Actions and permission for Actions to create pull requests, then run **Update WorkflowHub data** from the Actions tab. + +### Gem and remote-theme sites + +Gem and remote-theme installations do not copy GitHub workflow files into the website repository. After adding the WorkflowHub configuration above, add this small caller workflow to `.github/workflows/` in the website repository: + +```yml +name: Update WorkflowHub data + +on: + workflow_dispatch: + +permissions: + contents: write + pull-requests: write + +jobs: + update: + uses: ELIXIR-Belgium/elixir-toolkit-theme/.github/workflows/workflowhub.yml@main +``` + +Pin the reusable workflow to the same released theme tag used by the website instead of `main` for a production site. A `schedule` trigger can also be added if regular refreshes are useful. + +The repository must allow GitHub Actions to create pull requests under **Settings โ†’ Actions โ†’ General โ†’ Workflow permissions**. Run the caller manually from the Actions tab for the first update and review the resulting pull request before merging it. + +The generator validates required values and duplicate URLs. It also refuses to replace an existing non-empty data file with an empty response. Once automation is enabled, avoid editing the generated file manually because the next update will overwrite those changes. diff --git a/pages/example_pages/workflows.md b/pages/example_pages/workflows.md new file mode 100644 index 00000000..3adda757 --- /dev/null +++ b/pages/example_pages/workflows.md @@ -0,0 +1,18 @@ +--- +title: Workflows +datatable: true +--- + +This page demonstrates a searchable table populated from `_data/data_workflows.yaml`. See the [WorkflowHub workflows table documentation](workflowhub_integration) for manual and automated ways to maintain this file. + +## Listing workflows + +Use this snippet: + +```liquid +{% raw %}{% include workflow-table.html %}{% endraw %} +``` + +This renders as: + +{% include workflow-table.html %}