diff --git a/.github/workflows/best-practices-adherence-dev.yaml b/.github/workflows/best-practices-adherence-dev.yaml new file mode 100644 index 0000000..c05e05b --- /dev/null +++ b/.github/workflows/best-practices-adherence-dev.yaml @@ -0,0 +1,108 @@ +# This workflow is triggered by pushes the "backend" or "frontend" branch. +# It analyses the content of the "backend" or "frontent" folder, accordingly. + +name: Best Practices Adherence + +on: + push: + branches: + - backend + - frontend + workflow_dispatch: + +jobs: + analyze-and-report: + runs-on: ubuntu-latest + + steps: + # Clone the repository + - name: Checkout Code + uses: actions/checkout@v6 + with: + fetch-depth: 0 # Helps SonarQube to calculate "New Code" + + # Apply Ruff and store report + - name: Install Ruff + run: pip install ruff + + - name: Run Ruff Linting + env: + # Contains the name of the branch that triggered the workflow execution + BRANCH_NAME: ${{ github.ref_name }} + run: | + mkdir -p reports + ruff check --no-cache --exit-zero --verbose --output-format json --output-file reports/ruff-report.json $BRANCH_NAME + cat reports/ruff-report.json + continue-on-error: true # The workflow continues even if linting finds issues + + # Apply Coding Best Practices for Open Source Software + - name: Coding Best Practices for OSS + uses: EOEPCA/coding-best-practices-for-oss-action@main + with: + path-to-check: 'backend' + output-file: 'reports/coding-best-practices-report.json' + output-format: 'generic' + default-anchor-file: 'backend/coding-best-practices-issues.md' + + # Generate the SonarQube project properties + - name: SonarQube Project properties + env: + # Contains the name of the branch that triggered the workflow execution + BRANCH_NAME: ${{ github.ref_name }} + run: | + echo "Generating the Sonar project properties file ..." + echo "# Sonar Project Properties" > sonar-project.properties + echo "sonar.projectKey=${{ vars.EOEPCA_SONAR_PROJECT_KEY }}-$BRANCH_NAME" >> sonar-project.properties + echo "sonar.projectName=${{ vars.EOEPCA_SONAR_PROJECT_NAME }} ($BRANCH_NAME)" >> sonar-project.properties + echo "sonar.projectVersion=$GITHUB_SHA" >> sonar-project.properties + echo "sonar.sourceEncoding=UTF-8" >> sonar-project.properties + echo "sonar.projectBaseDir=." >> sonar-project.properties + echo "sonar.sources=$BRANCH_NAME" >> sonar-project.properties + echo "sonar.inclusions=${{ vars.EOEPCA_SONAR_INCLUSIONS }}" >> sonar-project.properties + echo "sonar.exclusions=${{ vars.EOEPCA_SONAR_EXCLUSIONS }}" >> sonar-project.properties + echo "sonar.python.version=3.11" >> sonar-project.properties + # --- Extra reports + echo "sonar.python.ruff.reportPaths=reports/ruff-report.json" >> sonar-project.properties + # --- Custom reports + echo "sonar.externalIssuesReportPaths=reports/coding-best-practices-report.json" >> sonar-project.properties + + cat sonar-project.properties + + # Run SonarQube Analysis (using the properties file generated above) + - name: SonarQube Scan + uses: sonarsource/sonarqube-scan-action@master + env: + SONAR_TOKEN: ${{ secrets.EOEPCA_SONAR_TOKEN }} + SONAR_HOST_URL: ${{ vars.EOEPCA_SONAR_HOST_URL }} + + # Wait for Quality Gate and Fetch Report + - name: Fetch SonarQube Quality Gate Result + shell: bash + run: | + # The scan action creates a metadata file we use to find the task URL + METADATA_FILE=".scannerwork/report-task.txt" + echo "SonarQube scanner metadata file:" + cat $METADATA_FILE + TASK_URL=$(grep "ceTaskUrl=" $METADATA_FILE | cut -d'=' -f2-) + + echo "Waiting for SonarQube background task to finish..." + + # Poll the API until the task is SUCCESS or FAILED + STATUS="PENDING" + while [ "$STATUS" == "" ] || [ "$STATUS" == "PENDING" ] || [ "$STATUS" == "IN_PROGRESS" ]; do + sleep 5 + RESPONSE=$(curl -s -u ${{ secrets.EOEPCA_SONAR_TOKEN }}: "$TASK_URL") + STATUS=$(echo $RESPONSE | jq -r '.task.status') + echo "Current Status: $STATUS" + done + + # Fetch the final status of the Quality Gate + ANALYSIS_ID=$(echo $RESPONSE | jq -r '.task.analysisId') + echo "Analysis ID: $ANALYSIS_ID" + GATE_URL="${{ vars.EOEPCA_SONAR_HOST_URL }}/api/qualitygates/project_status?analysisId=$ANALYSIS_ID" + echo "Quality Gate URL: $GATE_URL" + + curl -s -u ${{ secrets.EOEPCA_SONAR_TOKEN }}: "$GATE_URL" > reports/sonarqube-gate-report.json + + echo "Final report saved in reports/sonarqube-gate-report.json" + cat reports/sonarqube-gate-report.json | jq \ No newline at end of file diff --git a/backend/application_quality/settings.py b/backend/application_quality/settings.py index 0456e1a..f170fca 100644 --- a/backend/application_quality/settings.py +++ b/backend/application_quality/settings.py @@ -59,16 +59,20 @@ # Build the ALLOWED_HOSTS variable # PUBLIC_URL should contain a scheme, and possibly a port number -allowed_host = os.getenv("PUBLIC_URL") -if "://" not in allowed_host: - allowed_host = "http://" + allowed_host -parsed_allowed_host = urlparse(allowed_host) -allowed_domain = parsed_allowed_host.netloc -if ":" in allowed_domain: +public_url = os.getenv("PUBLIC_URL") +if "://" not in public_url: + public_url = "http://" + public_url +parsed_public_url = urlparse(public_url) +public_domain = parsed_public_url.netloc +if ":" in public_domain: # Remove the port number, if any - allowed_domain = allowed_domain.split(":")[0] + public_domain = public_domain.split(":")[0] -ALLOWED_HOSTS = [allowed_domain] +# BACKEND_SERVICE_HOST contains the internal k8s Service host name +# E.g. application-quality-api.application-quality.svc.cluster.local +service_host = os.getenv("BACKEND_SERVICE_HOST") + +ALLOWED_HOSTS = [public_domain, service_host] add_hosts_raw = os.getenv("ADDITIONAL_ALLOWED_HOSTS") if add_hosts_raw: add_hosts_list = [host.strip() for host in add_hosts_raw.split(',') if host.strip()] @@ -269,3 +273,8 @@ #CSRF_COOKIE_SECURE = False #CSRF_USE_SESSIONS = True CSRF_TRUSTED_ORIGINS = [os.getenv("PUBLIC_URL")] # API Base URL + +add_hosts_raw = os.getenv("ADDITIONAL_ALLOWED_HOSTS") +if add_hosts_raw: + add_hosts_list = [host.strip() for host in add_hosts_raw.split(',') if host.strip()] + CSRF_TRUSTED_ORIGINS.extend(add_hosts_list) \ No newline at end of file diff --git a/backend/backend/admin.py b/backend/backend/admin.py index 032b90d..f6822e8 100644 --- a/backend/backend/admin.py +++ b/backend/backend/admin.py @@ -7,7 +7,6 @@ from django_svelte_jsoneditor.widgets import SvelteJSONEditorWidget - class PipelineSettings(admin.ModelAdmin): list_display = ("name", "description", "owner", "version", "created_at", "edited_at") formfield_overrides = { @@ -38,9 +37,31 @@ class JobReportSettings(admin.ModelAdmin): JSONField: {"widget": SvelteJSONEditorWidget} } +class TriggerTypeSettings(admin.ModelAdmin): + list_display = ("__str__", "slug", "event_type_prefix", "status", "available") + formfield_overrides = { + JSONField: {"widget": SvelteJSONEditorWidget} + } + +class TriggerSettings(admin.ModelAdmin): + list_display = ("__str__", "owner", "trigger_type", "pipeline__name", "pipeline__version", "status", "enabled") + formfield_overrides = { + JSONField: {"widget": SvelteJSONEditorWidget} + } + +class TriggerEventSettings(admin.ModelAdmin): + list_display = ("event_time", "trigger__trigger_type__name", "event_type", "source", "trigger__slug", "trigger__pipeline__name", "trigger__pipeline__version", "pipeline_run") + formfield_overrides = { + JSONField: {"widget": SvelteJSONEditorWidget} + } + + admin.site.register(models.Pipeline, PipelineSettings) admin.site.register(models.PipelineRun, PipelineRunSettings) admin.site.register(models.Subworkflow, ToolSettings) admin.site.register(models.CommandLineTool, CommandSettings) admin.site.register(models.JobReport, JobReportSettings) +admin.site.register(models.TriggerType, TriggerTypeSettings) +admin.site.register(models.Trigger, TriggerSettings) +admin.site.register(models.TriggerEvent, TriggerEventSettings) admin.site.register(models.Tag) diff --git a/backend/backend/auth_backends.py b/backend/backend/auth_backends.py index 5f59f2d..67cad66 100644 --- a/backend/backend/auth_backends.py +++ b/backend/backend/auth_backends.py @@ -35,6 +35,7 @@ def update_user(self, user, claims): def logout_next_url(request): + logger.info("Received logout request: %s", request) url = ( f"{settings.OIDC_OP_USER_ENDPOINT.replace('userinfo', 'logout')}" "?response_type=code" diff --git a/backend/backend/fixtures/01-backend-tags.json b/backend/backend/fixtures/01-backend-tags.json new file mode 100644 index 0000000..3a6765f --- /dev/null +++ b/backend/backend/fixtures/01-backend-tags.json @@ -0,0 +1,65 @@ +[ + { + "model": "backend.tag", + "pk": 1, + "fields": { + "name": "asset: python" + } + }, + { + "model": "backend.tag", + "pk": 2, + "fields": { + "name": "asset: other" + } + }, + { + "model": "backend.tag", + "pk": 3, + "fields": { + "name": "asset: cwl" + } + }, + { + "model": "backend.tag", + "pk": 4, + "fields": { + "name": "asset: notebook" + } + }, + { + "model": "backend.tag", + "pk": 5, + "fields": { + "name": "type: best practice" + } + }, + { + "model": "backend.tag", + "pk": 6, + "fields": { + "name": "type: app quality" + } + }, + { + "model": "backend.tag", + "pk": 7, + "fields": { + "name": "type: app performance" + } + }, + { + "model": "backend.tag", + "pk": 8, + "fields": { + "name": "type: init" + } + }, + { + "model": "backend.tag", + "pk": 9, + "fields": { + "name": "asset: docker" + } + } +] \ No newline at end of file diff --git a/backend/backend/fixtures/02-backend-clts.json b/backend/backend/fixtures/02-backend-clts.json new file mode 100644 index 0000000..74b7314 --- /dev/null +++ b/backend/backend/fixtures/02-backend-clts.json @@ -0,0 +1,146 @@ +[ + { + "model": "backend.commandlinetool", + "pk": "ap_validator", + "fields": { + "name": "Application Package Validator", + "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: nexus.spaceapplications.com/repository/docker-eoepca/ap_validator:2025-03-05.1\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n ap-validator \\\\\r\n --format json \\\\\r\n --detail '$(inputs.detail)' \\\\\r\n --entry-point '$(inputs.entry_point)' \\\\\r\n '$(inputs.file_path)' > ~/ap_validator_report.json\r\n\r\n exit 0\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n file_path: string\r\n source_directory: Directory\r\n detail: string\r\n entry_point: string\r\n\r\n outputs:\r\n ap_validator_report:\r\n type: File\r\n outputBinding:\r\n glob: ap_validator_report.json\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: ap_validator_tool", + "version": "0.1" + } + }, + { + "model": "backend.commandlinetool", + "pk": "bandit", + "fields": { + "name": "Bandit", + "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: cytopia/bandit\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n PARAMS=\"-f $(inputs.output_format) -o $HOME/$(inputs.output_file)\"\r\n if [ \"$(inputs.exit_zero)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS --exit-zero\"\r\n fi\r\n if [ \"$(inputs.verbose)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -v\"\r\n fi\r\n\r\n bandit $PARAMS $(inputs.file_list.join(\" \"))\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n exit_zero:\r\n label: Exit with zero\r\n doc: Exit with 0, even with results found.\r\n type: boolean\r\n default: true\r\n file_list: string[]\r\n output_file:\r\n label: Output file\r\n doc: Write report to filename.\r\n type: string\r\n default: bandit_report.json\r\n output_format:\r\n label: Output format\r\n doc: Specify output format.\r\n type: string\r\n default: json\r\n source_directory: Directory\r\n verbose: boolean\r\n\r\n outputs:\r\n bandit_report:\r\n type: File\r\n outputBinding:\r\n glob: $(inputs.output_file)\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: bandit_tool", + "version": "0.1" + } + }, + { + "model": "backend.commandlinetool", + "pk": "clone", + "fields": { + "name": "Clone repo", + "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: alpine/git\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: clone_branch.sh\r\n entry: |-\r\n set -e\r\n\r\n if [ $(inputs.repo_branch) ]; then\r\n echo 'Branch specified: $(inputs.repo_branch). Cloning branch...'\r\n git clone $(inputs.repo_url) -b $(inputs.repo_branch)\r\n else\r\n echo 'No branch specified. Cloning default branch...'\r\n git clone $(inputs.repo_url)\r\n fi\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n repo_branch: string\r\n repo_url: string\r\n\r\n outputs:\r\n repo_directory:\r\n type: Directory\r\n outputBinding:\r\n glob: $(inputs.repo_url.split('/').pop().replace('.git',''))\r\n\r\n baseCommand: sh\r\n arguments:\r\n - clone_branch.sh\r\n id: clone_tool", + "version": "0.1" + } + }, + { + "model": "backend.commandlinetool", + "pk": "_digest", + "fields": { + "name": "Report Digest Generator", + "definition": "- class: CommandLineTool\r\n\r\n id: digest_generator\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: python:3.11-slim\r\n InlineJavascriptRequirement: {}\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: digest_pylint_report.py\r\n entry: |\r\n from datetime import datetime\r\n from typing import List, Dict, Any\r\n\r\n def digest(pylint_report: List[Dict[str, Any]], instance_name: str = \"\") -> Dict[str, Any]:\r\n \"\"\"\r\n Parses a JSON-decoded Pylint report array and aggregates it into the digest format.\r\n \"\"\"\r\n issue_counts = {\r\n \"info\": 0,\r\n \"convention\": 0,\r\n \"warning\": 0,\r\n \"error\": 0,\r\n \"security\": 0, # Pylint doesn't natively do securit\r\n \"critical\": 0\r\n }\r\n\r\n for issue in pylint_report:\r\n severity = issue.get(\"type\", \"\").lower()\r\n if severity == \"fatal\":\r\n issue_counts[\"critical\"] += 1\r\n elif severity == \"error\":\r\n issue_counts[\"error\"] += 1\r\n elif severity == \"warning\":\r\n issue_counts[\"warning\"] += 1\r\n elif severity == \"convention\":\r\n issue_counts[\"convention\"] += 1\r\n elif severity == \"refactor\":\r\n issue_counts[\"info\"] += 1 # Map code smells to info\r\n\r\n digest = {\r\n \"tool\": {\r\n \"name\": \"pylint\",\r\n \"instance\": instance_name,\r\n \"version\": \"\"\r\n },\r\n \"created_at\": datetime.now().isoformat(),\r\n \"issues\": issue_counts\r\n }\r\n return digest\r\n\r\n - entryname: digest_bandit_report.py\r\n entry: |\r\n from datetime import datetime\r\n from typing import Dict, Any\r\n\r\n def digest(bandit_report: Dict[str, Any], instance_name: str = \"\") -> Dict[str, Any]:\r\n \"\"\"\r\n Parses a JSON-decoded Bandit report array and aggregates it into the digest format.\r\n \"\"\"\r\n totals = bandit_report.get(\"metrics\", {}).get(\"_totals\", {})\r\n return {\r\n \"tool\": {\r\n \"name\": \"bandit\",\r\n \"instance\": instance_name,\r\n \"version\": \"\"\r\n },\r\n \"created_at\": datetime.now().isoformat(),\r\n \"issues\": {\r\n \"info\": totals.get(\"SEVERITY.LOW\", 0),\r\n \"convention\": 0,\r\n \"warning\": totals.get(\"SEVERITY.MEDIUM\", 0),\r\n \"error\": totals.get(\"SEVERITY.HIGH\", 0),\r\n \"security\": 0,\r\n \"critical\": 0,\r\n \"undefined\": totals.get(\"SEVERITY.UNDEFINED\", 0),\r\n }\r\n }\r\n\r\n - entryname: make_digest.py\r\n entry: |\r\n import json\r\n import sys\r\n from datetime import datetime\r\n from digest_pylint_report import digest as digest_pylint\r\n from digest_bandit_report import digest as digest_bandit\r\n #from digest_flake8_report import digest as digest_flake8\r\n #from digest_ruff_report import digest as digest_ruff\r\n\r\n tool_name = \"$(inputs.name)\"\r\n with open(\"$(inputs.report_file.path)\", \"r\", encoding=\"utf-8\") as file:\r\n # Load and parse the JSON file contents\r\n report_data = json.load(file)\r\n\r\n if tool_name == \"pylint\":\r\n digest_data = digest_pylint(report_data)\r\n elif tool_name == \"bandit\":\r\n digest_data = digest_bandit(report_data)\r\n #elif tool_name == \"flake8\":\r\n # digest_data = digest_flake8(report_data)\r\n #elif tool_name == \"ruff\":\r\n # digest_data = digest_ruff(report_data)\r\n else:\r\n digest_data = {\r\n \"tool\": {\r\n \"name\": tool_name,\r\n \"instance\": \"\",\r\n \"version\": \"n/a\"\r\n },\r\n \"created_at\": datetime.now().isoformat(),\r\n \"issues\": {\r\n \"info\": 0,\r\n \"convention\": 0,\r\n \"warning\": 0,\r\n \"error\": 0,\r\n \"security\": 0,\r\n \"critical\": 0\r\n }\r\n }\r\n\r\n try:\r\n with open(\"digest.json\", \"w\", encoding=\"utf-8\") as f:\r\n json.dump(digest_data, f, indent=2)\r\n print(\"[Digest] Successfully generated digest.json\")\r\n except Exception as e:\r\n print(f\"[Digest] Error writing file: {e}\", file=sys.stderr)\r\n sys.exit(1)\r\n\r\n baseCommand: [\"python3\", \"make_digest.py\"]\r\n\r\n inputs:\r\n name: string\r\n instance:\r\n type: string\r\n default: \"\"\r\n report_file: File\r\n\r\n outputs:\r\n digest_file:\r\n type: File\r\n outputBinding:\r\n glob: \"digest.json\"", + "version": "0.1" + } + }, + { + "model": "backend.commandlinetool", + "pk": "filter", + "fields": { + "name": "Filter", + "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: alpine:latest\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n find . -type f -regex \"$(inputs.regex)\" > $HOME/filter.out\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n regex: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n file_list:\r\n type: string[]\r\n outputBinding:\r\n glob: filter.out\r\n outputEval: |-\r\n $(self[0].contents.split('\\n').filter(function(line) {return line.trim() !== '';}))\r\n loadContents: true\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: filter_tool", + "version": "0.1" + } + }, + { + "model": "backend.commandlinetool", + "pk": "flake8", + "fields": { + "name": "Flake8", + "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: eoepca/appquality-flake8-json:v0.1.0\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n PARAMS=\"--format=$(inputs.output_format) --output-file=$HOME/$(inputs.output_file)\"\r\n if [ \"$(inputs.exit_zero)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS --exit-zero\"\r\n fi\r\n if [ \"$(inputs.verbose)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -v\"\r\n fi\r\n\r\n flake8 $PARAMS $(inputs.file_list.join(\" \"))\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n exit_zero:\r\n label: Exit with zero\r\n doc: Force Flake8 to use the exit status code 0 even if there are errors.\r\n type: boolean\r\n default: true\r\n file_list: string[]\r\n output_file:\r\n label: Output file\r\n doc: Redirect all output to the specified file.\r\n type: string\r\n default: flake8_report.json\r\n output_format:\r\n label: Output format\r\n doc: Select the formatter used to display errors to the user.\r\n type: string\r\n default: json\r\n source_directory: Directory\r\n verbose: boolean\r\n\r\n outputs:\r\n flake8_report:\r\n type: File\r\n outputBinding:\r\n glob: $(inputs.output_file)\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: flake8_tool", + "version": "0.1" + } + }, + { + "model": "backend.commandlinetool", + "pk": "notebook-bp-validator", + "fields": { + "name": "Jupyter Notebook Best Practices Validator", + "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: eoepca/notebook-bp-validator:2025-05-12.1\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n PARAMS=\"-s $(inputs.schema)\"\r\n if [ '$(inputs.abspath)' == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -p\"\r\n fi\r\n\r\n nb-validator $PARAMS $(inputs.file_list.join(\" \")) > ~/notebook-bp-validator_report.json\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n abspath: boolean\r\n file_list: string[]\r\n schema: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n notebook-bp-validator_report:\r\n type: File\r\n outputBinding:\r\n glob: notebook-bp-validator_report.json\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: notebook-bp-validator_tool", + "version": "0.1" + } + }, + { + "model": "backend.commandlinetool", + "pk": "papermill", + "fields": { + "name": "Papermill", + "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: continuumio/miniconda3\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n set -e\r\n\r\n cat ~/err.ipynb > ~/out.ipynb\r\n echo \"[Papermill] Source directory: $(inputs.source_directory.path)\"\r\n echo \"[Papermill] Notebook path: $(inputs.notebook_path)\"\r\n echo \"[Papermill] Extract requirements: $(inputs.extract_requirements)\"\r\n echo \"[Papermill] Extra requirements: $(inputs.extra_requirements)\"\r\n echo \"[Papermill] Execution parameters: $(inputs.execution_parameters)\"\r\n\r\n cd $(inputs.source_directory.path)\r\n\r\n if [ -s environment.yml ]; then\r\n echo \"Info: Found non-empty environment.yml file in the source folder. Will use it to setup the Conda environment.\"\r\n ENV=`grep \"name: \" environment.yml | cut - -d' ' -f2`\r\n echo \"Info: Environment name: $ENV\"\r\n cp -p environment.yml ~/environment.yml\r\n touch ~/requirements.txt\r\n else\r\n if [ \"$(inputs.extract_requirements)\" = \"true\" ]; then\r\n echo \"Info: Generating requirements.txt\"\r\n cd ~\r\n apt update && apt install -y jq\r\n pip install pipreqsnb\r\n pipreqsnb --force --encoding utf-8 --savepath ~/requirements.txt $(inputs.source_directory.path)/$(inputs.notebook_path)\r\n echo \"Info: Generating environment.yml\"\r\n ENV=`cat $(inputs.source_directory.path)/$(inputs.notebook_path) | jq .metadata.kernelspec.name -r`\r\n echo \"Info: Environment name: $ENV\"\r\n echo \"name: $ENV\" > ~/environment.yml\r\n else\r\n echo \"Info: Generating environment.yml\"\r\n pip install pyyaml\r\n python ~/script.py $(inputs.notebook_path) | tee ~/environment.yml\r\n touch ~/requirements.txt\r\n fi\r\n fi\r\n\r\n if [ ! -s ~/environment.yml ]; then\r\n echo \"Error: environment.yml is empty. Aborting environment creation.\"\r\n exit 1\r\n fi\r\n\r\n echo \"Requirements file:\"\r\n cat ~/requirements.txt\r\n\r\n echo \"Environment file:\"\r\n cat ~/environment.yml\r\n\r\n ENV=`grep \"name: \" ~/environment.yml | cut - -d' ' -f2`\r\n echo \"Info: Environment name: $ENV\"\r\n\r\n conda env create -f ~/environment.yml\r\n conda run -n $ENV pip install ipykernel papermill\r\n conda run -n $ENV pip install -r ~/requirements.txt\r\n if [ \"$(inputs.extra_requirements)\" != \"\" ]; then\r\n conda run -n $ENV pip install $(inputs.extra_requirements)\r\n fi\r\n conda run -n $ENV ipython kernel install --user --name $ENV\r\n conda run -n $ENV papermill $(inputs.source_directory.path)/$(inputs.notebook_path) ~/out.ipynb\r\n\r\n - entryname: script.py\r\n entry: |-\r\n import json\r\n import yaml\r\n import sys\r\n\r\n try:\r\n with open('$(inputs.notebook_path)') as f:\r\n nb_json = json.load(f)\r\n except Exception as e:\r\n sys.stderr.write(f\"Failed to load notebook: {e}\\n\")\r\n sys.exit(1)\r\n\r\n conda_env = nb_json.get(\"metadata\", {}).get(\"software_requirements\", {}).get(\"conda_environment\", None)\r\n\r\n try:\r\n kernel_name = nb_json[\"metadata\"][\"kernelspec\"][\"name\"]\r\n except KeyError as e:\r\n sys.stderr.write(f\"Missing key in notebook metadata: {e}\\n\")\r\n sys.exit(1)\r\n\r\n if conda_env:\r\n if conda_env.get(\"name\", None) == kernel_name:\r\n if conda_env.get(\"dependencies\"):\r\n res = yaml.dump({\"name\": conda_env[\"name\"], \"dependencies\": conda_env[\"dependencies\"]})\r\n print(res)\r\n else:\r\n sys.stderr.write(\"No dependencies found in the environment specification.\\n\")\r\n sys.exit(1)\r\n else:\r\n sys.stderr.write(\"Kernel name does not match the conda environment name.\\n\")\r\n sys.exit(1)\r\n else:\r\n res = yaml.dump({\"name\": kernel_name, \"dependencies\": []})\r\n print(res)\r\n\r\n - entryname: err.ipynb\r\n entry: |-\r\n {\r\n \"cells\": [\r\n {\r\n \"cell_type\": \"markdown\",\r\n \"metadata\": {},\r\n \"source\": [\r\n \"Error: The notebook could not be run.\"\r\n ]\r\n }\r\n ],\r\n \"metadata\": {\r\n \"language_info\": {\r\n \"name\": \"python\"\r\n }\r\n },\r\n \"nbformat\": 4,\r\n \"nbformat_minor\": 2\r\n }\r\n\r\n inputs:\r\n notebook_path: string\r\n extra_requirements:\r\n label: Python requirements\r\n doc: Space-separated list of Python libraries to be installed before executing the notebook\r\n type: string\r\n default: \"\"\r\n extract_requirements:\r\n label: Extract requirements\r\n doc: Inspect the notebook cells to derive library requirements\r\n type: boolean\r\n default: false\r\n execution_parameters:\r\n label: Execution parameters\r\n doc: These parameters are provided as inputs to the executed notebooks\r\n type: string\r\n default: \"\"\r\n source_directory: Directory\r\n\r\n outputs:\r\n output_nb:\r\n type: File\r\n outputBinding:\r\n glob: out.ipynb\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: papermill_tool", + "version": "0.1" + } + }, + { + "model": "backend.commandlinetool", + "pk": "pylint", + "fields": { + "name": "Pylint", + "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: cytopia/pylint\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n PARAMS=\"--output-format=$(inputs.output_format) --output=$HOME/$(inputs.output_file) --disable=$(inputs.disable)\"\r\n if [ \"$(inputs.exit_zero)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS --exit-zero\"\r\n fi\r\n if [ \"$(inputs.errors_only)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -E\"\r\n fi\r\n if [ \"$(inputs.verbose)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -v\"\r\n fi\r\n\r\n pylint $PARAMS $(inputs.file_list.join(\" \"))\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n disable: string\r\n errors_only: boolean\r\n exit_zero:\r\n doc: |-\r\n Always return a 0 (non-error) status code, even if lint errors are found. This is primarily useful in continuous integration scripts.\r\n type: boolean\r\n default: true\r\n file_list: string[]\r\n output_file:\r\n doc: Specify an output file.\r\n type: string\r\n default: pylint_report.json\r\n output_format:\r\n doc: |-\r\n Set the output format. Available formats are: text, parseable, colorized, json2 (improved json format), json (old json format) and msvs (visual studio). You can also give a reporter class, e.g. mypackage.mymodule.MyReporterClass.\r\n type: string\r\n default: json\r\n source_directory: Directory\r\n verbose: boolean\r\n\r\n outputs:\r\n pylint_report:\r\n type: File\r\n outputBinding:\r\n glob: $(inputs.output_file)\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: pylint_tool", + "version": "0.1" + } + }, + { + "model": "backend.commandlinetool", + "pk": "ruff", + "fields": { + "name": "Ruff", + "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: ghcr.io/astral-sh/ruff:alpine\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n PARAMS=\"--output-format $(inputs.output_format) -o $HOME/$(inputs.output_file)\"\r\n if [ \"$(inputs.exit_zero)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -e\"\r\n fi\r\n if [ \"$(inputs.no_cache)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -n\"\r\n fi\r\n if [ \"$(inputs.verbose)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -v\"\r\n fi\r\n\r\n ruff check $PARAMS $(inputs.file_list.join(\" \"))\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n exit_zero:\r\n label: Exit with zero\r\n doc: Exit with status code \"0\", even upon detecting lint violations.\r\n type: boolean\r\n default: true\r\n no_cache:\r\n label: Disable cache\r\n doc: Disable cache reads.\r\n type: boolean\r\n default: true\r\n file_list: string[]\r\n output_file:\r\n label: Output file\r\n doc: Specify file to write the linter output to.\r\n type: string\r\n default: ruff_report.json\r\n output_format:\r\n label: Output format\r\n doc: |-\r\n Output serialization format for violations. Possible values: concise, full, json, json-lines, junit, grouped, github, gitlab, pylint, rdjson, azure, sarif.\r\n type: string\r\n default: json\r\n source_directory: Directory\r\n verbose: boolean\r\n\r\n outputs:\r\n ruff_report:\r\n type: File\r\n outputBinding:\r\n glob: $(inputs.output_file)\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: ruff_tool", + "version": "0.1" + } + }, + { + "model": "backend.commandlinetool", + "pk": "save", + "fields": { + "name": "Save report", + "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: curlimages/curl\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n name: string\r\n instance:\r\n type: string\r\n default: \"\"\r\n pipeline_id: string\r\n report: File\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n baseCommand: curl\r\n arguments:\r\n - prefix: -X\r\n valueFrom: POST\r\n - prefix: -L\r\n valueFrom: |-\r\n $(inputs.server_url + '/api/pipelines/' + inputs.pipeline_id + '/runs/' + inputs.run_id + '/jobreports/?name=' + inputs.name + '&instance=' + inputs.instance)\r\n - prefix: -H\r\n valueFrom: Content-Type:application/json\r\n - prefix: -d\r\n valueFrom: $('@' + inputs.report.path)\r\n id: save_tool", + "version": "0.1" + } + }, + { + "model": "backend.commandlinetool", + "pk": "save_report_and_digest", + "fields": { + "name": "Save report and digest", + "definition": "- class: CommandLineTool\r\n id: save_tool\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: curlimages/curl\r\n InlineJavascriptRequirement: {}\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: save_script.sh\r\n entry: |-\r\n #!/usr/bin/env sh\r\n set -e\r\n\r\n URL=\"$(inputs.server_url)/api/pipelines/$(inputs.pipeline_id)/runs/$(inputs.run_id)/jobreports/?name=$(inputs.name)&instance=$(inputs.instance)\"\r\n echo \"[Save] Backend request URL: $URL\"\r\n REPORT_FILE=\"$(inputs.report_file.path)\"\r\n DIGEST_FILE=\"$(inputs.digest_file === null ? 'digest.json' : inputs.digest_file.path)\"\r\n echo \"[Save] Mandatory report file: $REPORT_FILE\"\r\n echo \"[Save] Optional digest file: $DIGEST_FILE\"\r\n\r\n # Check if the digest file does not exist OR is empty (size 0)\r\n if [ ! -s \"$DIGEST_FILE\" ]; then\r\n echo \"[Save] Digest file is missing or empty. Creating it with an empty dictionary...\"\r\n echo '{}' > \"$DIGEST_FILE\"\r\n fi\r\n\r\n # Execute the curl command\r\n echo \"[Save] Sending POST request to $URL...\"\r\n curl -X POST -F \"report=@$REPORT_FILE\" -F \"digest=@$DIGEST_FILE\" \"$URL\"\r\n\r\n baseCommand: [\"sh\", \"save_script.sh\"]\r\n\r\n inputs:\r\n name: string\r\n instance:\r\n type: string\r\n default: \"\"\r\n pipeline_id: string\r\n report_file: File\r\n digest_file: File?\r\n run_id: string\r\n server_url: string\r\n\r\n outputs:\r\n stdout: stdout", + "version": "0.1" + } + }, + { + "model": "backend.commandlinetool", + "pk": "sonarqube_get_report", + "fields": { + "name": "[SonarQube] Get report", + "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: curlimages/curl\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n sonarqube_project_key:\r\n type: string\r\n sonarqube_server:\r\n type: string\r\n default: sonarqube-sonarqube.sonarqube:9000\r\n sonarqube_token:\r\n type: string\r\n\r\n outputs:\r\n sonarqube_project_key:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_project_key)\r\n sonarqube_report:\r\n type: File\r\n outputBinding:\r\n glob: sonarqube_report.json\r\n sonarqube_server:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_server)\r\n sonarqube_token:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_token)\r\n stdout: sonarqube_report.json\r\n\r\n baseCommand:\r\n - curl\r\n arguments:\r\n - prefix: -L\r\n valueFrom: |-\r\n $('http://' + inputs.sonarqube_server + '/api/issues/search?components=' + inputs.sonarqube_project_key)\r\n - prefix: -u\r\n valueFrom: $(inputs.sonarqube_token + ':')\r\n id: sonarqube_get_report_tool", + "version": "0.1" + } + }, + { + "model": "backend.commandlinetool", + "pk": "sonarqube_init", + "fields": { + "name": "[SonarQube] Init", + "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: curlimages/curl\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n sonarqube_project_key:\r\n type: string\r\n sonarqube_project_name:\r\n type: string\r\n sonarqube_server:\r\n type: string\r\n default: sonarqube-sonarqube.sonarqube:9000\r\n sonarqube_token:\r\n type: string\r\n\r\n outputs:\r\n sonarqube_project_key:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_project_key)\r\n sonarqube_server:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_server)\r\n sonarqube_token:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_token)\r\n\r\n baseCommand:\r\n - curl\r\n arguments:\r\n - prefix: -X\r\n valueFrom: POST\r\n - prefix: -L\r\n valueFrom: $('http://' + inputs.sonarqube_server + '/api/projects/create')\r\n - prefix: -u\r\n valueFrom: $(inputs.sonarqube_token + ':')\r\n - prefix: -d\r\n valueFrom: $('name=' + inputs.sonarqube_project_name)\r\n - prefix: -d\r\n valueFrom: $('project=' + inputs.sonarqube_project_key)\r\n id: sonarqube_init_tool", + "version": "0.1" + } + }, + { + "model": "backend.commandlinetool", + "pk": "sonarqube_scan", + "fields": { + "name": "[SonarQube] Scan repo", + "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: sonarsource/sonar-scanner-cli\r\n EnvVarRequirement:\r\n envDef:\r\n SONAR_HOST_URL: $('http://' + inputs.sonarqube_server)\r\n SONAR_TOKEN: $(inputs.sonarqube_token)\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n sonarqube_project_key:\r\n type: string\r\n sonarqube_server:\r\n type: string\r\n default: sonarqube-sonarqube.sonarqube:9000\r\n sonarqube_token:\r\n type: string\r\n source_directory:\r\n type: Directory\r\n\r\n outputs:\r\n sonarqube_project_key:\r\n type: string\r\n outputBinding:\r\n glob:\r\n outputEval: $(inputs.sonarqube_project_key)\r\n sonarqube_server:\r\n type: string\r\n outputBinding:\r\n glob:\r\n outputEval: $(inputs.sonarqube_server)\r\n sonarqube_token:\r\n type: string\r\n outputBinding:\r\n glob:\r\n outputEval: $(inputs.sonarqube_token)\r\n\r\n baseCommand:\r\n - sonar-scanner\r\n arguments:\r\n - prefix: -D\r\n valueFrom: $('sonar.projectKey=' + inputs.sonarqube_project_key)\r\n separate: false\r\n # - prefix: -D\r\n # valueFrom: $('sonar.userHome=$HOME')\r\n # separate: false\r\n - prefix: -D\r\n valueFrom: $('sonar.projectBaseDir=' + inputs.source_directory.path + '/../')\r\n separate: false\r\n # - prefix: -D\r\n # valueFrom: $('sonar.source=$HOME' /*+ inputs.source_directory.path*/)\r\n # separate: false\r\n - prefix: -X\r\n id: sonarqube_scan_tool", + "version": "0.1" + } + }, + { + "model": "backend.commandlinetool", + "pk": "trivy", + "fields": { + "name": "Trivy", + "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: aquasec/trivy:latest\r\n\r\n inputs:\r\n image: string\r\n\r\n outputs:\r\n trivy_report:\r\n type: File\r\n outputBinding:\r\n glob: trivy_report.json\r\n\r\n baseCommand: trivy\r\n arguments:\r\n - image\r\n - prefix: -f\r\n valueFrom: json\r\n - prefix: -o\r\n valueFrom: trivy_report.json\r\n - $(inputs.image)\r\n id: trivy_tool", + "version": "0.1" + } + } +] \ No newline at end of file diff --git a/backend/backend/fixtures/03-backend-tools.json b/backend/backend/fixtures/03-backend-tools.json new file mode 100644 index 0000000..83e6a7e --- /dev/null +++ b/backend/backend/fixtures/03-backend-tools.json @@ -0,0 +1,449 @@ +[ + { + "model": "backend.subworkflow", + "pk": "ap_validator_subworkflow", + "fields": { + "name": "Application Package Validator", + "description": "Validation tool for checking OGC compliance of CWL files for application packages.", + "pipeline_step": "ap_validator_subworkflow:\r\n in:\r\n ap_validator.detail: ap_validator_subworkflow.ap_validator.detail\r\n ap_validator.entry_point: ap_validator_subworkflow.ap_validator.entry_point\r\n filter.regex: ap_validator_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#ap_validator_subworkflow'\r\n out: []", + "definition": "- class: Workflow\r\n\r\n requirements:\r\n ScatterFeatureRequirement: {}\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: ap_validator\r\n ap_validator.detail: string\r\n ap_validator.entry_point: string\r\n filter.regex: string\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n []\r\n\r\n steps:\r\n filter_ap_validator_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n ap_validator_step:\r\n in:\r\n file_path: filter_ap_validator_step/file_list\r\n source_directory: source_directory\r\n detail: ap_validator.detail\r\n entry_point: ap_validator.entry_point\r\n scatter: file_path\r\n run: '#ap_validator_tool'\r\n out:\r\n - ap_validator_report\r\n save_ap_validator_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: ap_validator_step/ap_validator_report\r\n run_id: run_id\r\n server_url: server_url\r\n scatter: report\r\n run: '#save_tool'\r\n out: []\r\n id: ap_validator_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", + "user_params": { + "filter": { + "regex": { + "type": "string", + "label": "regex", + "default": ".*\\.cwl" + } + }, + "ap_validator": { + "detail": { + "doc": "Output detail (none|errors|hints|all). Default: hints", + "type": "string", + "label": "Detail", + "default": "hints" + }, + "entry_point": { + "doc": "Name of entry point (Workflow or CommandLineTool)", + "type": "string", + "label": "Entry point", + "default": "main" + } + } + }, + "version": "0.1", + "tags": [ + 3, + 5 + ], + "tools": [ + "filter", + "ap_validator", + "save" + ] + } + }, + { + "model": "backend.subworkflow", + "pk": "bandit_subworkflow", + "fields": { + "name": "Bandit", + "description": "Bandit - Bandit is a tool designed to find common security issues in Python code", + "pipeline_step": "bandit_subworkflow:\r\n in:\r\n bandit.verbose: bandit_subworkflow.bandit.verbose\r\n filter.regex: bandit_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#bandit_subworkflow'\r\n out: []", + "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: bandit\r\n bandit.verbose: boolean\r\n filter.regex: string\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n bandit_report:\r\n type: File\r\n outputSource: bandit_step/bandit_report\r\n\r\n steps:\r\n filter_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n bandit_step:\r\n in:\r\n file_list: filter_step/file_list\r\n source_directory: source_directory\r\n verbose: bandit.verbose\r\n run: '#bandit_tool'\r\n out:\r\n - bandit_report\r\n digest_step:\r\n run: '#digest_generator'\r\n in:\r\n name: name\r\n report_file: bandit_step/bandit_report\r\n out:\r\n - digest_file\r\n save_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report_file: bandit_step/bandit_report\r\n digest_file: digest_step/digest_file\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: bandit_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", + "user_params": { + "bandit": { + "verbose": { + "doc": "Output extra information like excluded and included files.", + "type": "boolean", + "label": "Verbose", + "default": false + } + }, + "filter": { + "regex": { + "type": "string", + "label": "regex", + "default": ".*\\.py" + } + } + }, + "version": "0.1", + "status": "Stable", + "available": true, + "tags": [ + 1, + 6 + ], + "tools": [ + "filter", + "bandit", + "_digest", + "save_report_and_digest" + ] + } + }, + { + "model": "backend.subworkflow", + "pk": "clone_subworkflow", + "fields": { + "name": "Clone repo", + "description": "git-clone - Clone a repository into a new directory", + "pipeline_step": "clone_step:\r\n in:\r\n clone.repo_branch: clone_subworkflow.clone.repo_branch\r\n clone.repo_url: clone_subworkflow.clone.repo_url\r\n run: '#clone_subworkflow'\r\n out:\r\n - repo_directory", + "definition": "- class: Workflow\r\n\r\n inputs:\r\n clone.repo_branch:\r\n type: string\r\n default: ''\r\n clone.repo_url: string\r\n\r\n outputs:\r\n repo_directory:\r\n type: Directory\r\n outputSource: clone_tool_step/repo_directory\r\n\r\n steps:\r\n clone_tool_step:\r\n in:\r\n repo_branch: clone.repo_branch\r\n repo_url: clone.repo_url\r\n run: '#clone_tool'\r\n out:\r\n - repo_directory\r\n id: clone_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", + "user_params": { + "clone": { + "repo_url": { + "doc": "URL to the repository to clone.", + "type": "string", + "label": "Repo URL", + "default": "" + }, + "repo_branch": { + "doc": "Branch to checkout instead of the remote HEAD.", + "type": "string", + "label": "Repo branch", + "default": "" + } + } + }, + "version": "0.1", + "tags": [ + 2, + 8 + ], + "tools": [ + "clone" + ] + } + }, + { + "model": "backend.subworkflow", + "pk": "flake8_subworkflow", + "fields": { + "name": "Flake8", + "description": "flake8 - Style guide enforcement tool for Python", + "pipeline_step": "flake8_subworkflow:\r\n in:\r\n filter.regex: flake8_subworkflow.filter.regex\r\n flake8.verbose: flake8_subworkflow.flake8.verbose\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#flake8_subworkflow'\r\n out: []", + "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: flake8\r\n filter.regex: string\r\n flake8.verbose: boolean\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n flake8_report:\r\n type: File\r\n outputSource: flake8_step/flake8_report\r\n\r\n steps:\r\n filter_step:\r\n run: '#filter_tool'\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n out:\r\n - file_list\r\n flake8_step:\r\n run: '#flake8_tool'\r\n in:\r\n file_list: filter_step/file_list\r\n source_directory: source_directory\r\n verbose: flake8.verbose\r\n out:\r\n - flake8_report\r\n digest_step:\r\n run: '#digest_generator'\r\n in:\r\n name: name\r\n report_file: flake8_step/flake8_report\r\n out:\r\n - digest_file\r\n save_step:\r\n run: '#save_tool'\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report_file: flake8_step/flake8_report\r\n digest_file: digest_step/digest_file\r\n run_id: run_id\r\n server_url: server_url\r\n out: []\r\n id: flake8_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", + "user_params": { + "filter": { + "regex": { + "type": "string", + "label": "regex", + "default": ".*\\.py" + } + }, + "flake8": { + "verbose": { + "doc": "Increase the verbosity of Flake8’s output.", + "type": "boolean", + "label": "Verbose", + "default": false + } + } + }, + "version": "0.1", + "status": "Stable", + "available": true, + "tags": [ + 1, + 5 + ], + "tools": [ + "_digest", + "filter", + "flake8", + "save_report_and_digest" + ] + } + }, + { + "model": "backend.subworkflow", + "pk": "notebook-bp-validator_subworkflow", + "fields": { + "name": "Jupyter Notebook Best Practices Validator", + "description": "This tool aims at validating the notebooks against the CEOS Jupyter Notebook Best Practice v1.1 document.", + "pipeline_step": "notebook-bp-validator_subworkflow:\r\n in:\r\n filter.regex: notebook-bp-validator_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n notebook-bp-validator.abspath: notebook-bp-validator_subworkflow.notebook-bp-validator.abspath\r\n notebook-bp-validator.schema: notebook-bp-validator_subworkflow.notebook-bp-validator.schema\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#notebook-bp-validator_subworkflow'\r\n out: []", + "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: notebook-bp-validator\r\n filter.regex: string\r\n pipeline_id: string\r\n notebook-bp-validator.abspath: boolean\r\n notebook-bp-validator.schema: string\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n notebook-bp-validator_report:\r\n type: File\r\n outputSource: notebook-bp-validator_step/notebook-bp-validator_report\r\n\r\n steps:\r\n filter_notebook-bp-validator_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n notebook-bp-validator_step:\r\n in:\r\n abspath: notebook-bp-validator.abspath \r\n file_list: filter_notebook-bp-validator_step/file_list\r\n source_directory: source_directory\r\n schema: notebook-bp-validator.schema\r\n run: '#notebook-bp-validator_tool'\r\n out:\r\n - notebook-bp-validator_report\r\n save_notebook-bp-validator_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: notebook-bp-validator_step/notebook-bp-validator_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: notebook-bp-validator_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", + "user_params": { + "filter": { + "regex": { + "type": "string", + "label": "regex", + "default": ".*\\.ipynb" + } + }, + "notebook-bp-validator": { + "schema": { + "doc": "Supported values: 'eumetsat' or 'schema.org'", + "type": "string", + "label": "Schema", + "default": "eumetsat" + }, + "abspath": { + "doc": "Uses absolute paths in output.", + "type": "boolean", + "label": "Absolute path", + "default": false + } + } + }, + "version": "0.1", + "tags": [ + 4, + 5 + ], + "tools": [ + "filter", + "notebook-bp-validator", + "save" + ] + } + }, + { + "model": "backend.subworkflow", + "pk": "papermill_subworkflow", + "fields": { + "name": "Papermill", + "description": "Papermill is a tool for parameterizing and executing Jupyter Notebooks.", + "pipeline_step": "papermill_subworkflow:\r\n in:\r\n filter.regex: papermill_subworkflow.filter.regex\r\n papermill.extract_requirements: papermill_subworkflow.papermill.extract_requirements\r\n papermill.extra_requirements: papermill_subworkflow.papermill.extra_requirements\r\n papermill.execution_parameters: papermill_subworkflow.papermill.execution_parameters\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#papermill_subworkflow'\r\n out: []", + "definition": "- class: Workflow\r\n\r\n # --- This workflow runs Papermill on a single notebook and saves the result.\r\n\r\n id: papermill_execution_subworkflow\r\n\r\n inputs:\r\n notebook_path: string # Single notebook file\r\n\r\n # Inputs for papermill_tool\r\n source_directory: Directory\r\n extract_requirements: boolean?\r\n extra_requirements: string?\r\n execution_parameters: string?\r\n\r\n # Inputs for save_tool\r\n name: string\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n papermill_step:\r\n in:\r\n notebook_path: notebook_path\r\n extract_requirements: extract_requirements\r\n extra_requirements: extra_requirements\r\n execution_parameters: execution_parameters\r\n source_directory: source_directory\r\n run: '#papermill_tool'\r\n out:\r\n - output_nb\r\n\r\n save_papermill_step:\r\n in:\r\n name: name\r\n instance: notebook_path\r\n pipeline_id: pipeline_id\r\n report: papermill_step/output_nb\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n\r\n- class: Workflow\r\n\r\n # --- This workflow filters the input files then executes the above sub-workflow on each of them (scatter)\r\n\r\n id: papermill_subworkflow\r\n\r\n requirements:\r\n ScatterFeatureRequirement: {}\r\n\r\n inputs:\r\n filter.regex: string\r\n papermill.extract_requirements: boolean\r\n papermill.extra_requirements: string\r\n papermill.execution_parameters: string\r\n name:\r\n type: string\r\n default: papermill\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs: []\r\n\r\n steps:\r\n filter_papermill_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list # Array of strings (the input for scatter)\r\n\r\n execute_notebooks_step:\r\n # Run the subworkflow defined above\r\n run: '#papermill_execution_subworkflow'\r\n # Scatter the above sub-workflow [Papermill => Save] over the array of files\r\n scatter: notebook_path\r\n in:\r\n # The list of notebooks filtered in the previous step\r\n notebook_path: filter_papermill_step/file_list\r\n # Other sub-workflow inputs\r\n source_directory: source_directory\r\n extract_requirements: papermill.extract_requirements\r\n extra_requirements: papermill.extra_requirements\r\n execution_parameters: papermill.execution_parameters\r\n name: name\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n\r\n out: []\r\n\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", + "user_params": { + "filter": { + "regex": { + "type": "string", + "label": "regex", + "default": ".*\\.ipynb" + } + }, + "papermill": { + "extra_requirements": { + "doc": "Python requirements", + "type": "string", + "label": "Python requirements", + "default": "" + }, + "execution_parameters": { + "doc": "E.g. alpha=0.6, ratio=0.1", + "type": "string", + "label": "Execution parameters", + "default": "" + }, + "extract_requirements": { + "doc": "Extract requirements", + "type": "boolean", + "label": "Extract requirements", + "default": false + } + } + }, + "version": "0.1", + "tags": [ + 4, + 7 + ], + "tools": [ + "filter", + "papermill", + "save" + ] + } + }, + { + "model": "backend.subworkflow", + "pk": "pylint_subworkflow", + "fields": { + "name": "Pylint", + "description": "pylint - Static code analyser tool for Python", + "pipeline_step": "pylint_subworkflow:\r\n in:\r\n filter.regex: pylint_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n pylint.disable: pylint_subworkflow.pylint.disable\r\n pylint.errors_only: pylint_subworkflow.pylint.errors_only\r\n pylint.verbose: pylint_subworkflow.pylint.verbose\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#pylint_subworkflow'\r\n out: []", + "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: pylint\r\n filter.regex: string\r\n pipeline_id: string\r\n pylint.disable: string\r\n pylint.errors_only: boolean\r\n pylint.verbose: boolean\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n pylint_report:\r\n type: File\r\n outputSource: pylint_step/pylint_report\r\n\r\n steps:\r\n filter_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n pylint_step:\r\n in:\r\n disable: pylint.disable\r\n errors_only: pylint.errors_only\r\n file_list: filter_step/file_list\r\n source_directory: source_directory\r\n verbose: pylint.verbose\r\n run: '#pylint_tool'\r\n out:\r\n - pylint_report\r\n digest_step:\r\n run: '#digest_generator'\r\n in:\r\n name: name\r\n report_file: pylint_step/pylint_report\r\n out:\r\n - digest_file\r\n save_step:\r\n run: '#save_tool'\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report_file: pylint_step/pylint_report\r\n digest_file: digest_step/digest_file\r\n run_id: run_id\r\n server_url: server_url\r\n out: []\r\n id: pylint_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", + "user_params": { + "filter": { + "regex": { + "type": "string", + "label": "regex", + "default": ".*\\.py" + } + }, + "pylint": { + "disable": { + "doc": "Disable the message, report, category or checker with the given id(s).", + "type": "string", + "label": "Disable IDs", + "default": "E0401" + }, + "verbose": { + "doc": "In verbose mode, extra non-checker-related info will be displayed.", + "type": "boolean", + "label": "Verbose", + "default": false + }, + "errors_only": { + "doc": "In error mode, messages with a category besides ERROR or FATAL are suppressed, and no reports are done by default. Error mode is compatible with disabling specific errors.", + "type": "boolean", + "label": "Errors only", + "default": false + } + } + }, + "version": "0.1", + "status": "Stable", + "available": true, + "tags": [ + 1, + 5 + ], + "tools": [ + "filter", + "pylint", + "_digest", + "save_report_and_digest" + ] + } + }, + { + "model": "backend.subworkflow", + "pk": "ruff_ipynb_subworkflow", + "fields": { + "name": "Ruff - Notebook", + "description": "Ruff - An extremely fast Python linter and code formatter, written in Rust", + "pipeline_step": "ruff_ipynb_subworkflow:\r\n in:\r\n filter.regex: ruff_ipynb_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n ruff.verbose: ruff_ipynb_subworkflow.ruff.verbose\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#ruff_ipynb_subworkflow'\r\n out: []", + "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: ruff\r\n filter.regex: string\r\n pipeline_id: string\r\n ruff.verbose: boolean\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n ruff_report:\r\n type: File\r\n outputSource: ruff_step/ruff_report\r\n\r\n steps:\r\n filter_ruff_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n ruff_step:\r\n in:\r\n file_list: filter_ruff_step/file_list\r\n source_directory: source_directory\r\n verbose: ruff.verbose\r\n run: '#ruff_tool'\r\n out:\r\n - ruff_report\r\n save_ruff_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: ruff_step/ruff_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: ruff_ipynb_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", + "user_params": { + "ruff": { + "verbose": { + "doc": "Enable verbose logging.", + "type": "boolean", + "label": "Verbose", + "default": false + } + }, + "filter": { + "regex": { + "type": "string", + "label": "regex", + "default": ".*\\.ipynb" + } + } + }, + "version": "0.1", + "tags": [ + 4, + 5 + ], + "tools": [ + "filter", + "ruff", + "save" + ] + } + }, + { + "model": "backend.subworkflow", + "pk": "ruff_subworkflow", + "fields": { + "name": "Ruff", + "description": "Ruff - An extremely fast Python linter and code formatter, written in Rust", + "pipeline_step": "ruff_subworkflow:\r\n in:\r\n filter.regex: ruff_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n ruff.verbose: ruff_subworkflow.ruff.verbose\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#ruff_subworkflow'\r\n out: []", + "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: ruff\r\n filter.regex: string\r\n pipeline_id: string\r\n ruff.verbose: boolean\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n ruff_report:\r\n type: File\r\n outputSource: ruff_step/ruff_report\r\n\r\n steps:\r\n filter_ruff_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n ruff_step:\r\n in:\r\n file_list: filter_ruff_step/file_list\r\n source_directory: source_directory\r\n verbose: ruff.verbose\r\n run: '#ruff_tool'\r\n out:\r\n - ruff_report\r\n save_ruff_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: ruff_step/ruff_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: ruff_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", + "user_params": { + "ruff": { + "verbose": { + "doc": "Enable verbose logging.", + "type": "boolean", + "label": "Verbose", + "default": false + } + }, + "filter": { + "regex": { + "type": "string", + "label": "regex", + "default": ".*\\.py" + } + } + }, + "version": "0.1", + "tags": [ + 1, + 5 + ], + "tools": [ + "filter", + "ruff", + "save" + ] + } + }, + { + "model": "backend.subworkflow", + "pk": "sonarqube_subworkflow", + "fields": { + "name": "SonarQube", + "description": "SonarQube - Code Quality, Security & Static Analysis Tool\r\n\r\nThis tool creates a project in our internal SonarQube server, sends it the code for analysis, and then retrieves the analysis results for storage in the database.", + "pipeline_step": "sonarqube_subworkflow:\r\n in:\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n\r\n # \"sonarqube_init\" in the subworkflow receives all the inputs\r\n sonarqube_project_key: sonarqube_subworkflow.sonarqube_init.sonarqube_project_key\r\n sonarqube_project_name: sonarqube_subworkflow.sonarqube_init.sonarqube_project_name\r\n sonarqube_server: sonarqube_subworkflow.sonarqube_init.sonarqube_server\r\n sonarqube_token: sonarqube_subworkflow.sonarqube_init.sonarqube_token\r\n\r\n run: '#sonarqube_subworkflow'\r\n out: []", + "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: sonarqube\r\n pipeline_id:\r\n type: string\r\n run_id:\r\n type: string\r\n server_url:\r\n type: string\r\n source_directory:\r\n type: Directory\r\n sonarqube_project_key:\r\n type: string\r\n sonarqube_project_name:\r\n type: string\r\n sonarqube_server:\r\n type: string\r\n default: sonarqube-sonarqube.sonarqube:9000\r\n sonarqube_token:\r\n type: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n\r\n sonarqube_init_step:\r\n in:\r\n sonarqube_project_key: sonarqube_project_key\r\n sonarqube_project_name: sonarqube_project_name\r\n sonarqube_server: sonarqube_server\r\n sonarqube_token: sonarqube_token\r\n run: '#sonarqube_init_tool'\r\n out:\r\n - sonarqube_project_key\r\n - sonarqube_server\r\n - sonarqube_token\r\n\r\n sonarqube_scan_step:\r\n in:\r\n sonarqube_project_key: sonarqube_init_step/sonarqube_project_key\r\n sonarqube_project_name: sonarqube_project_name\r\n sonarqube_server: sonarqube_server\r\n sonarqube_token: sonarqube_token\r\n source_directory: source_directory\r\n run: '#sonarqube_scan_tool'\r\n out:\r\n - sonarqube_project_key\r\n - sonarqube_server\r\n - sonarqube_token\r\n\r\n sonarqube_get_report_step:\r\n in:\r\n sonarqube_project_key: sonarqube_scan_step/sonarqube_project_key\r\n sonarqube_server: sonarqube_server\r\n sonarqube_token: sonarqube_token\r\n run: '#sonarqube_get_report_tool'\r\n out:\r\n - sonarqube_project_key\r\n - sonarqube_server\r\n - sonarqube_token\r\n - sonarqube_report\r\n\r\n save_sonarqube_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: sonarqube_get_report_step/sonarqube_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n\r\n id: sonarqube_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", + "user_params": { + "sonarqube_init": { + "sonarqube_token": { + "type": "string", + "label": "Sonarqube Access Token", + "default": "123" + }, + "sonarqube_server": { + "type": "string", + "label": "Sonarqube Server", + "default": "sonarqube-sonarqube.sonarqube:9000" + }, + "sonarqube_project_key": { + "type": "string", + "label": "Sonarqube Project Key", + "default": "test_project_01" + }, + "sonarqube_project_name": { + "type": "string", + "label": "Sonarqube Project Name", + "default": "Test Project 01" + } + } + }, + "version": "0.1", + "status": "Stable", + "available": true, + "tags": [], + "tools": [ + "save", + "sonarqube_get_report", + "sonarqube_init", + "sonarqube_scan" + ] + } + }, + { + "model": "backend.subworkflow", + "pk": "trivy_subworkflow", + "fields": { + "name": "Trivy", + "description": "The all-in-one open source security scanner\r\nUse Trivy to find vulnerabilities (CVE) & misconfigurations (IaC) across code repositories, binary artifacts, container images, Kubernetes clusters, and more.", + "pipeline_step": "trivy_subworkflow:\r\n in:\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n trivy.image: trivy_subworkflow.trivy.image\r\n run: '#trivy_subworkflow'\r\n out: []", + "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: trivy\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n trivy.image: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n save_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: trivy_step/trivy_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n trivy_step:\r\n in:\r\n image: trivy.image\r\n run: '#trivy_tool'\r\n out:\r\n - trivy_report\r\n id: trivy_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", + "user_params": { + "trivy": { + "image": { + "type": "string", + "label": "Docker image", + "default": "alpine/git" + } + } + }, + "version": "0.1", + "tags": [ + 6, + 9 + ], + "tools": [ + "save", + "trivy" + ] + } + } +] \ No newline at end of file diff --git a/backend/backend/fixtures/04-backend-pipelines.json b/backend/backend/fixtures/04-backend-pipelines.json new file mode 100644 index 0000000..6dee06c --- /dev/null +++ b/backend/backend/fixtures/04-backend-pipelines.json @@ -0,0 +1,231 @@ +[ + { + "model": "backend.pipeline", + "pk": 6, + "fields": { + "name": "Python pipeline", + "description": "Runs a series of static analysis tools on python files", + "template": "#!/usr/bin/env cwltool\r\n\r\n$graph:\r\n- class: Workflow\r\n\r\n requirements:\r\n SubworkflowFeatureRequirement: {}\r\n\r\n inputs:\r\n{%- for tool in subworkflows %}\r\n {%- for tool_name, params in tool.user_params.items() %}\r\n {%- for param_name, param in params.items() %}\r\n {{ tool.slug }}.{{ tool_name }}.{{ param_name }}:\r\n label: \"{{ param.label or param_name }}\"\r\n {%- if param.doc %}\r\n doc: |-\r\n {{ param.doc }}\r\n {%- endif %}\r\n type: {{ param.type }}\r\n default: {{ param.default | tojson }}\r\n {%- endfor %}\r\n {%- endfor %}\r\n{%- endfor %}\r\n branch:\r\n type: string\r\n default: ''\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n {% for tool in subworkflows %}{{ tool.pipeline_step | indent(4) }}\r\n {% endfor %}\r\n id: main\r\n{% for tool in subworkflows %}\r\n{{ tool.definition }}\r\n{% endfor %}\r\ncwlVersion: v1.2", + "default_inputs": {}, + "owner": null, + "created_at": null, + "edited_at": null, + "version": "0.1", + "tools": [ + "clone_subworkflow", + "bandit_subworkflow", + "flake8_subworkflow", + "pylint_subworkflow", + "ruff_subworkflow" + ] + } + }, + { + "model": "backend.pipeline", + "pk": 8, + "fields": { + "name": "Notebook pipeline", + "description": "Runs a static analysis and then executes Jupyter Notebooks files (ipynb).", + "template": "#!/usr/bin/env cwltool\r\n\r\n$graph:\r\n- class: Workflow\r\n\r\n requirements:\r\n SubworkflowFeatureRequirement: {}\r\n\r\n inputs:\r\n{%- for tool in subworkflows %}\r\n {%- for tool_name, params in tool.user_params.items() %}\r\n {%- for param_name, param in params.items() %}\r\n {{ tool.slug }}.{{ tool_name }}.{{ param_name }}:\r\n label: \"{{ param.label or param_name }}\"\r\n {%- if param.doc %}\r\n doc: |-\r\n {{ param.doc }}\r\n {%- endif %}\r\n type: {{ param.type }}\r\n default: {{ param.default | tojson }}\r\n {%- endfor %}\r\n {%- endfor %}\r\n{%- endfor %}\r\n branch:\r\n type: string\r\n default: ''\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n {% for tool in subworkflows %}{{ tool.pipeline_step | indent(4) }}\r\n {% endfor %}\r\n id: main\r\n{% for tool in subworkflows %}\r\n{{ tool.definition }}\r\n{% endfor %}\r\ncwlVersion: v1.2", + "default_inputs": {}, + "owner": null, + "created_at": null, + "edited_at": null, + "version": "0.1", + "tools": [ + "clone_subworkflow", + "notebook-bp-validator_subworkflow", + "papermill_subworkflow", + "ruff_ipynb_subworkflow" + ] + } + }, + { + "model": "backend.pipeline", + "pk": 11, + "fields": { + "name": "Docker pipeline", + "description": "Finds vulnerabilities and misconfigurations in Docker images.", + "template": "#!/usr/bin/env cwltool\r\n\r\n$graph:\r\n- class: Workflow\r\n\r\n requirements:\r\n SubworkflowFeatureRequirement: {}\r\n\r\n inputs:\r\n{%- for tool in subworkflows %}\r\n {%- for tool_name, params in tool.user_params.items() %}\r\n {%- for param_name, param in params.items() %}\r\n {{ tool.slug }}.{{ tool_name }}.{{ param_name }}:\r\n label: \"{{ param.label or param_name }}\"\r\n {%- if param.doc %}\r\n doc: |-\r\n {{ param.doc }}\r\n {%- endif %}\r\n type: {{ param.type }}\r\n default: {{ param.default | tojson }}\r\n {%- endfor %}\r\n {%- endfor %}\r\n{%- endfor %}\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n {% for tool in subworkflows %}{{ tool.pipeline_step | indent(4) }}\r\n {% endfor %}\r\n id: main\r\n{% for tool in subworkflows %}\r\n{{ tool.definition }}\r\n{% endfor %}\r\ncwlVersion: v1.2", + "default_inputs": {}, + "owner": null, + "created_at": null, + "edited_at": null, + "version": "0.1", + "tools": [ + "trivy_subworkflow" + ] + } + }, + { + "model": "backend.pipeline", + "pk": 17, + "fields": { + "name": "Notebook static analysis pipeline", + "description": "Runs static analysis on Jupyer Notebook files (ipynb)", + "template": "#!/usr/bin/env cwltool\r\n\r\n$graph:\r\n- class: Workflow\r\n\r\n requirements:\r\n SubworkflowFeatureRequirement: {}\r\n\r\n inputs:\r\n{%- for tool in subworkflows %}\r\n {%- for tool_name, params in tool.user_params.items() %}\r\n {%- for param_name, param in params.items() %}\r\n {{ tool.slug }}.{{ tool_name }}.{{ param_name }}:\r\n label: \"{{ param.label or param_name }}\"\r\n {%- if param.doc %}\r\n doc: |-\r\n {{ param.doc }}\r\n {%- endif %}\r\n type: {{ param.type }}\r\n default: {{ param.default | tojson }}\r\n {%- endfor %}\r\n {%- endfor %}\r\n{%- endfor %}\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n {% for tool in subworkflows %}{{ tool.pipeline_step | indent(4) }}\r\n {% endfor %}\r\n id: main\r\n{% for tool in subworkflows %}\r\n{{ tool.definition }}\r\n{% endfor %}\r\ncwlVersion: v1.2", + "default_inputs": {}, + "owner": null, + "created_at": "2025-09-16T07:47:34.320Z", + "edited_at": "2025-11-18T10:55:05.144Z", + "version": "0.1", + "tools": [ + "clone_subworkflow", + "notebook-bp-validator_subworkflow" + ] + } + }, + { + "model": "backend.pipeline", + "pk": 18, + "fields": { + "name": "Ruff for python", + "description": "Ruff for python", + "template": "#!/usr/bin/env cwltool\n\n$graph:\n- class: Workflow\n\n requirements:\n SubworkflowFeatureRequirement: {}\n\n inputs:\n{%- for tool in subworkflows %}\n {%- for tool_name, params in tool.user_params.items() %}\n {%- for param_name, param in params.items() %}\n {{ tool.slug }}.{{ tool_name }}.{{ param_name }}:\n label: \"{{ param.label or param_name }}\"\n {%- if param.doc %}\n doc: |-\n {{ param.doc }}\n {%- endif %}\n type: {{ param.type }}\n default: {{ param.default | tojson }}\n {%- endfor %}\n {%- endfor %}\n{%- endfor %}\n pipeline_id: string\n run_id: string\n server_url: string\n\n outputs: []\n\n steps:\n {% for tool in subworkflows %}{{ tool.pipeline_step | indent(4) }}\n {% endfor %}\n id: main\n{% for tool in subworkflows %}\n{{ tool.definition }}\n{% endfor %}\ncwlVersion: v1.2\n", + "default_inputs": {}, + "owner": null, + "created_at": "2025-10-21T00:42:55.449Z", + "edited_at": "2025-10-21T00:42:55.449Z", + "version": "0.1", + "tools": [ + "clone_subworkflow", + "ruff_subworkflow" + ] + } + }, + { + "model": "backend.pipeline", + "pk": 19, + "fields": { + "name": "Notebook execution pipeline", + "description": "Executes selected notebooks using Papermill", + "template": "#!/usr/bin/env cwltool\r\n\r\n$graph:\r\n- class: Workflow\r\n\r\n requirements:\r\n SubworkflowFeatureRequirement: {}\r\n\r\n inputs:\r\n{%- for tool in subworkflows %}\r\n {%- for tool_name, params in tool.user_params.items() %}\r\n {%- for param_name, param in params.items() %}\r\n {{ tool.slug }}.{{ tool_name }}.{{ param_name }}:\r\n label: \"{{ param.label or param_name }}\"\r\n {%- if param.doc %}\r\n doc: |-\r\n {{ param.doc }}\r\n {%- endif %}\r\n type: {{ param.type }}\r\n default: {{ param.default | tojson }}\r\n {%- endfor %}\r\n {%- endfor %}\r\n{%- endfor %}\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n {% for tool in subworkflows %}{{ tool.pipeline_step | indent(4) }}\r\n {% endfor %}\r\n id: main\r\n{% for tool in subworkflows %}\r\n{{ tool.definition }}\r\n{% endfor %}\r\ncwlVersion: v1.2", + "default_inputs": {}, + "owner": null, + "created_at": "2025-11-04T11:59:57.031Z", + "edited_at": "2025-11-25T12:05:05.664Z", + "version": "0.1", + "tools": [ + "papermill_subworkflow", + "clone_subworkflow" + ] + } + }, + { + "model": "backend.pipeline", + "pk": 33, + "fields": { + "name": "Notebook BP Validation", + "description": "Notebook BP Validation", + "template": "#!/usr/bin/env cwltool\n\n$graph:\n- class: Workflow\n\n requirements:\n SubworkflowFeatureRequirement: {}\n\n inputs:\n{%- for tool in subworkflows %}\n {%- for tool_name, params in tool.user_params.items() %}\n {%- for param_name, param in params.items() %}\n {{ tool.slug }}.{{ tool_name }}.{{ param_name }}:\n label: \"{{ param.label or param_name }}\"\n {%- if param.doc %}\n doc: |-\n {{ param.doc }}\n {%- endif %}\n type: {{ param.type }}\n default: {{ param.default | tojson }}\n {%- endfor %}\n {%- endfor %}\n{%- endfor %}\n pipeline_id: string\n run_id: string\n server_url: string\n\n outputs: []\n\n steps:\n {% for tool in subworkflows %}{{ tool.pipeline_step | indent(4) }}\n {% endfor %}\n id: main\n{% for tool in subworkflows %}\n{{ tool.definition }}\n{% endfor %}\ncwlVersion: v1.0", + "default_inputs": { + "clone_subworkflow": { + "clone": { + "repo_url": { + "doc": "URL to the repository to clone.", + "type": "string", + "label": "Repo URL", + "default": "https://github.com/SpaceApplications/deepesdl-doc.git" + }, + "repo_branch": { + "doc": "Branch to checkout instead of the remote HEAD.", + "type": "string", + "label": "Repo branch", + "default": "main" + } + } + }, + "notebook-bp-validator_subworkflow": { + "filter": { + "regex": { + "type": "string", + "label": "regex", + "default": ".*00.*\\\\.ipynb" + } + }, + "notebook-bp-validator": { + "schema": { + "doc": "Supported values: 'eumetsat' or 'schema.org'", + "type": "string", + "label": "Schema", + "default": "eumetsat" + }, + "abspath": { + "doc": "Uses absolute paths in output.", + "type": "boolean", + "label": "Absolute path", + "default": false + } + } + } + }, + "owner": 1, + "created_at": "2025-05-12T12:05:00.360Z", + "edited_at": "2025-05-13T08:29:18.845Z", + "version": "0.1", + "tools": [ + "clone_subworkflow", + "notebook-bp-validator_subworkflow" + ] + } + }, + { + "model": "backend.pipeline", + "pk": 34, + "fields": { + "name": "Sonarqube", + "description": "Sonarqube", + "template": "#!/usr/bin/env cwltool\r\n\r\n$graph:\r\n- class: Workflow\r\n\r\n requirements:\r\n SubworkflowFeatureRequirement: {}\r\n\r\n inputs:\r\n{%- for tool in subworkflows %}\r\n {%- for tool_name, params in tool.user_params.items() %}\r\n {%- for param_name, param in params.items() %}\r\n {{ tool.slug }}.{{ tool_name }}.{{ param_name }}:\r\n label: \"{{ param.label or param_name }}\"\r\n {%- if param.doc %}\r\n doc: |-\r\n {{ param.doc }}\r\n {%- endif %}\r\n type: {{ param.type }}\r\n default: {{ param.default | tojson }}\r\n {%- endfor %}\r\n {%- endfor %}\r\n{%- endfor %}\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n {% for tool in subworkflows %}{{ tool.pipeline_step | indent(4) }}\r\n {% endfor %}\r\n id: main\r\n{% for tool in subworkflows %}\r\n{{ tool.definition }}\r\n{% endfor %}\r\ncwlVersion: v1.2", + "default_inputs": { + "clone_subworkflow": { + "clone": { + "repo_url": { + "doc": "URL to the repository to clone.", + "type": "string", + "label": "Repo URL", + "default": "https://github.com/EOEPCA/application-quality" + }, + "repo_branch": { + "doc": "Branch to checkout instead of the remote HEAD.", + "type": "string", + "label": "Repo branch", + "default": "backend" + } + } + }, + "sonarqube_subworkflow": { + "sonarqube_init": { + "sonarqube_token": { + "type": "string", + "label": "Sonarqube Access Token", + "default": "sqa_6bc1a5a84dac182e8c714c0c8d3ccbcf7b97c0a2" + }, + "sonarqube_server": { + "type": "string", + "label": "Sonarqube Server", + "default": "application-quality-sonarqube-sonarqube.application-quality-sonarqube.svc.cluster.local:9000" + }, + "sonarqube_project_key": { + "type": "string", + "label": "Sonarqube Project Key", + "default": "application_quality_bb" + }, + "sonarqube_project_name": { + "type": "string", + "label": "Sonarqube Project Name", + "default": "Application Quality BB" + } + } + } + }, + "owner": 4, + "created_at": "2026-03-24T10:59:05.290Z", + "edited_at": "2026-03-25T19:10:55.447Z", + "version": "0.1", + "tools": [ + "clone_subworkflow", + "sonarqube_subworkflow" + ] + } + } +] \ No newline at end of file diff --git a/backend/backend/fixtures/05-backend-triggertypes.json b/backend/backend/fixtures/05-backend-triggertypes.json new file mode 100644 index 0000000..9d9f387 --- /dev/null +++ b/backend/backend/fixtures/05-backend-triggertypes.json @@ -0,0 +1,74 @@ +[ + { + "model": "backend.triggertype", + "pk": "github_webhook_ping", + "fields": { + "description": "Ping events originating from a GitHub repository", + "name": "GitHub Ping Event (for testing webhooks)", + "event_type_prefix": "org.eoepca.webhook.github.ping", + "data": {}, + "status": "Testing", + "available": true + } + }, + { + "model": "backend.triggertype", + "pk": "github_webhook_pull_request", + "fields": { + "description": "Pull Request events (open and synchronize) originating from a GitHub repository", + "name": "GitHub Pull Request", + "event_type_prefix": "org.eoepca.webhook.github.pull_request", + "data": {}, + "status": "Testing", + "available": true + } + }, + { + "model": "backend.triggertype", + "pk": "github_webhook_push", + "fields": { + "description": "Push events originating from a GitHub repository", + "name": "GitHub Push Event", + "event_type_prefix": "org.eoepca.webhook.github.push", + "data": {}, + "status": "Testing", + "available": true + } + }, + { + "model": "backend.triggertype", + "pk": "gitlab_webhook_merge_request", + "fields": { + "description": "Merge Request events originating from a GitLab repository", + "name": "GitLab Push Event", + "event_type_prefix": "org.eoepca.webhook.gitlab.merge_request", + "data": {}, + "status": "Testing", + "available": true + } + }, + { + "model": "backend.triggertype", + "pk": "gitlab_webhook_ping", + "fields": { + "description": "Ping events originating from a GitLab repository", + "name": "GitLab Ping Event (for testing webhooks)", + "event_type_prefix": "org.eoepca.webhook.gitlab.ping", + "data": {}, + "status": "Testing", + "available": true + } + }, + { + "model": "backend.triggertype", + "pk": "gitlab_webhook_push", + "fields": { + "description": "Push events originating from a GitLab repository", + "name": "GitLab Push Event", + "event_type_prefix": "org.eoepca.webhook.gitlab.push", + "data": {}, + "status": "Testing", + "available": true + } + } +] \ No newline at end of file diff --git a/backend/backend/fixtures/2025-10-17.json b/backend/backend/fixtures/2025-10-17.json deleted file mode 100644 index 68e2014..0000000 --- a/backend/backend/fixtures/2025-10-17.json +++ /dev/null @@ -1,642 +0,0 @@ -[ - { - "model": "backend.pipeline", - "pk": 6, - "fields": { - "name": "Python pipeline", - "description": "Runs a series of static analysis tools on python files", - "template": "#!/usr/bin/env cwltool\r\n\r\n$graph:\r\n- class: Workflow\r\n\r\n requirements:\r\n SubworkflowFeatureRequirement: {}\r\n\r\n inputs:\r\n{%- for tool in subworkflows %}\r\n {%- for tool_name, params in tool.user_params.items() %}\r\n {%- for param_name, param in params.items() %}\r\n {{ tool.slug }}.{{ tool_name }}.{{ param_name }}:\r\n label: \"{{ param.label or param_name }}\"\r\n {%- if param.doc %}\r\n doc: |-\r\n {{ param.doc }}\r\n {%- endif %}\r\n type: {{ param.type }}\r\n default: {{ param.default | tojson }}\r\n {%- endfor %}\r\n {%- endfor %}\r\n{%- endfor %}\r\n branch:\r\n type: string\r\n default: ''\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n {% for tool in subworkflows %}{{ tool.pipeline_step | indent(4) }}\r\n {% endfor %}\r\n id: main\r\n{% for tool in subworkflows %}\r\n{{ tool.definition }}\r\n{% endfor %}\r\ncwlVersion: v1.2", - "default_inputs": {}, - "owner": null, - "created_at": null, - "edited_at": null, - "version": "0.1", - "tools": [ - "clone_subworkflow", - "bandit_subworkflow", - "flake8_subworkflow", - "pylint_subworkflow", - "ruff_subworkflow" - ] - } - }, - { - "model": "backend.pipeline", - "pk": 8, - "fields": { - "name": "Notebook pipeline", - "description": "Runs a static analysis and then executes Jupyter Notebooks files (ipynb).", - "template": "#!/usr/bin/env cwltool\r\n\r\n$graph:\r\n- class: Workflow\r\n\r\n requirements:\r\n SubworkflowFeatureRequirement: {}\r\n\r\n inputs:\r\n{%- for tool in subworkflows %}\r\n {%- for tool_name, params in tool.user_params.items() %}\r\n {%- for param_name, param in params.items() %}\r\n {{ tool.slug }}.{{ tool_name }}.{{ param_name }}:\r\n label: \"{{ param.label or param_name }}\"\r\n {%- if param.doc %}\r\n doc: |-\r\n {{ param.doc }}\r\n {%- endif %}\r\n type: {{ param.type }}\r\n default: {{ param.default | tojson }}\r\n {%- endfor %}\r\n {%- endfor %}\r\n{%- endfor %}\r\n branch:\r\n type: string\r\n default: ''\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n {% for tool in subworkflows %}{{ tool.pipeline_step | indent(4) }}\r\n {% endfor %}\r\n id: main\r\n{% for tool in subworkflows %}\r\n{{ tool.definition }}\r\n{% endfor %}\r\ncwlVersion: v1.2", - "default_inputs": {}, - "owner": null, - "created_at": null, - "edited_at": null, - "version": "0.1", - "tools": [ - "clone_subworkflow", - "notebook-bp-validator_subworkflow", - "papermill_subworkflow", - "ruff_ipynb_subworkflow" - ] - } - }, - { - "model": "backend.pipeline", - "pk": 11, - "fields": { - "name": "Docker pipeline", - "description": "Finds vulnerabilities and misconfigurations in Docker images.", - "template": "#!/usr/bin/env cwltool\r\n\r\n$graph:\r\n- class: Workflow\r\n\r\n requirements:\r\n SubworkflowFeatureRequirement: {}\r\n\r\n inputs:\r\n{%- for tool in subworkflows %}\r\n {%- for tool_name, params in tool.user_params.items() %}\r\n {%- for param_name, param in params.items() %}\r\n {{ tool.slug }}.{{ tool_name }}.{{ param_name }}:\r\n label: \"{{ param.label or param_name }}\"\r\n {%- if param.doc %}\r\n doc: |-\r\n {{ param.doc }}\r\n {%- endif %}\r\n type: {{ param.type }}\r\n default: {{ param.default | tojson }}\r\n {%- endfor %}\r\n {%- endfor %}\r\n{%- endfor %}\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n {% for tool in subworkflows %}{{ tool.pipeline_step | indent(4) }}\r\n {% endfor %}\r\n id: main\r\n{% for tool in subworkflows %}\r\n{{ tool.definition }}\r\n{% endfor %}\r\ncwlVersion: v1.2", - "default_inputs": {}, - "owner": null, - "created_at": null, - "edited_at": null, - "version": "0.1", - "tools": [ - "trivy_subworkflow" - ] - } - }, - { - "model": "backend.tag", - "pk": 1, - "fields": { - "name": "asset: python" - } - }, - { - "model": "backend.tag", - "pk": 2, - "fields": { - "name": "asset: other" - } - }, - { - "model": "backend.tag", - "pk": 3, - "fields": { - "name": "asset: cwl" - } - }, - { - "model": "backend.tag", - "pk": 4, - "fields": { - "name": "asset: notebook" - } - }, - { - "model": "backend.tag", - "pk": 5, - "fields": { - "name": "type: best practice" - } - }, - { - "model": "backend.tag", - "pk": 6, - "fields": { - "name": "type: app quality" - } - }, - { - "model": "backend.tag", - "pk": 7, - "fields": { - "name": "type: app performance" - } - }, - { - "model": "backend.tag", - "pk": 8, - "fields": { - "name": "type: init" - } - }, - { - "model": "backend.tag", - "pk": 9, - "fields": { - "name": "asset: docker" - } - }, - { - "model": "backend.subworkflow", - "pk": "ap_validator_subworkflow", - "fields": { - "name": "Application Package Validator", - "description": "Validation tool for checking OGC compliance of CWL files for application packages.", - "pipeline_step": "ap_validator_subworkflow:\r\n in:\r\n ap_validator.detail: ap_validator_subworkflow.ap_validator.detail\r\n ap_validator.entry_point: ap_validator_subworkflow.ap_validator.entry_point\r\n filter.regex: ap_validator_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#ap_validator_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n requirements:\r\n ScatterFeatureRequirement: {}\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: ap_validator\r\n ap_validator.detail: string\r\n ap_validator.entry_point: string\r\n filter.regex: string\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n []\r\n\r\n steps:\r\n filter_ap_validator_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n ap_validator_step:\r\n in:\r\n file_path: filter_ap_validator_step/file_list\r\n source_directory: source_directory\r\n detail: ap_validator.detail\r\n entry_point: ap_validator.entry_point\r\n scatter: file_path\r\n run: '#ap_validator_tool'\r\n out:\r\n - ap_validator_report\r\n save_ap_validator_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: ap_validator_step/ap_validator_report\r\n run_id: run_id\r\n server_url: server_url\r\n scatter: report\r\n run: '#save_tool'\r\n out: []\r\n id: ap_validator_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "filter": { - "regex": { - "type": "string", - "label": "regex", - "default": ".*\\.cwl" - } - }, - "ap_validator": { - "detail": { - "doc": "Output detail (none|errors|hints|all). Default: hints", - "type": "string", - "label": "Detail", - "default": "hints" - }, - "entry_point": { - "doc": "Name of entry point (Workflow or CommandLineTool)", - "type": "string", - "label": "Entry point", - "default": "main" - } - } - }, - "version": "0.1", - "tags": [ - 3, - 5 - ], - "tools": [ - "ap_validator", - "filter", - "save" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "bandit_subworkflow", - "fields": { - "name": "Bandit", - "description": "Bandit - Bandit is a tool designed to find common security issues in Python code", - "pipeline_step": "bandit_subworkflow:\r\n in:\r\n bandit.verbose: bandit_subworkflow.bandit.verbose\r\n filter.regex: bandit_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#bandit_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: bandit\r\n bandit.verbose: boolean\r\n filter.regex: string\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n bandit_report:\r\n type: File\r\n outputSource: bandit_step/bandit_report\r\n\r\n steps:\r\n bandit_step:\r\n in:\r\n file_list: filter_bandit_step/file_list\r\n source_directory: source_directory\r\n verbose: bandit.verbose\r\n run: '#bandit_tool'\r\n out:\r\n - bandit_report\r\n filter_bandit_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n save_bandit_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: bandit_step/bandit_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: bandit_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "bandit": { - "verbose": { - "doc": "Output extra information like excluded and included files.", - "type": "boolean", - "label": "Verbose", - "default": false - } - }, - "filter": { - "regex": { - "type": "string", - "label": "regex", - "default": ".*\\.py" - } - } - }, - "version": "0.1", - "tags": [ - 1, - 6 - ], - "tools": [ - "bandit", - "filter", - "save" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "clone_subworkflow", - "fields": { - "name": "Clone repo", - "description": "git-clone - Clone a repository into a new directory", - "pipeline_step": "clone_step:\r\n in:\r\n clone.repo_branch: clone_subworkflow.clone.repo_branch\r\n clone.repo_url: clone_subworkflow.clone.repo_url\r\n run: '#clone_subworkflow'\r\n out:\r\n - repo_directory", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n clone.repo_branch:\r\n type: string\r\n default: ''\r\n clone.repo_url: string\r\n\r\n outputs:\r\n repo_directory:\r\n type: Directory\r\n outputSource: clone_tool_step/repo_directory\r\n\r\n steps:\r\n clone_tool_step:\r\n in:\r\n repo_branch: clone.repo_branch\r\n repo_url: clone.repo_url\r\n run: '#clone_tool'\r\n out:\r\n - repo_directory\r\n id: clone_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "clone": { - "repo_url": { - "doc": "URL to the repository to clone.", - "type": "string", - "label": "Repo URL", - "default": "" - }, - "repo_branch": { - "doc": "Branch to checkout instead of the remote HEAD.", - "type": "string", - "label": "Repo branch", - "default": "" - } - } - }, - "version": "0.1", - "tags": [ - 2, - 8 - ], - "tools": [ - "clone" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "flake8_subworkflow", - "fields": { - "name": "Flake8", - "description": "flake8 - Style guide enforcement tool for Python", - "pipeline_step": "flake8_subworkflow:\r\n in:\r\n filter.regex: flake8_subworkflow.filter.regex\r\n flake8.verbose: flake8_subworkflow.flake8.verbose\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#flake8_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: flake8\r\n filter.regex: string\r\n flake8.verbose: boolean\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n flake8_report:\r\n type: File\r\n outputSource: flake8_step/flake8_report\r\n\r\n steps:\r\n filter_flake8_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n flake8_step:\r\n in:\r\n file_list: filter_flake8_step/file_list\r\n source_directory: source_directory\r\n verbose: flake8.verbose\r\n run: '#flake8_tool'\r\n out:\r\n - flake8_report\r\n save_flake8_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: flake8_step/flake8_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: flake8_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "filter": { - "regex": { - "type": "string", - "label": "regex", - "default": ".*\\.py" - } - }, - "flake8": { - "verbose": { - "doc": "Increase the verbosity of Flake8’s output.", - "type": "boolean", - "label": "Verbose", - "default": false - } - } - }, - "version": "0.1", - "tags": [ - 1, - 5 - ], - "tools": [ - "filter", - "flake8", - "save" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "notebook-bp-validator_subworkflow", - "fields": { - "name": "Jupyter Notebook Best Practices Validator", - "description": "This tool aims at validating the notebooks against the CEOS Jupyter Notebook Best Practice v1.1 document.", - "pipeline_step": "notebook-bp-validator_subworkflow:\r\n in:\r\n filter.regex: notebook-bp-validator_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n notebook-bp-validator.abspath: notebook-bp-validator_subworkflow.notebook-bp-validator.abspath\r\n notebook-bp-validator.schema: notebook-bp-validator_subworkflow.notebook-bp-validator.schema\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#notebook-bp-validator_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: notebook-bp-validator\r\n filter.regex: string\r\n pipeline_id: string\r\n notebook-bp-validator.abspath: boolean\r\n notebook-bp-validator.schema: string\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n notebook-bp-validator_report:\r\n type: File\r\n outputSource: notebook-bp-validator_step/notebook-bp-validator_report\r\n\r\n steps:\r\n filter_notebook-bp-validator_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n notebook-bp-validator_step:\r\n in:\r\n abspath: notebook-bp-validator.abspath \r\n file_list: filter_notebook-bp-validator_step/file_list\r\n source_directory: source_directory\r\n schema: notebook-bp-validator.schema\r\n run: '#notebook-bp-validator_tool'\r\n out:\r\n - notebook-bp-validator_report\r\n save_notebook-bp-validator_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: notebook-bp-validator_step/notebook-bp-validator_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: notebook-bp-validator_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "filter": { - "regex": { - "type": "string", - "label": "regex", - "default": ".*\\.ipynb" - } - }, - "notebook-bp-validator": { - "schema": { - "doc": "Supported values: 'eumetsat' or 'schema.org'", - "type": "string", - "label": "Schema", - "default": "eumetsat" - }, - "abspath": { - "doc": "Uses absolute paths in output.", - "type": "boolean", - "label": "Absolute path", - "default": false - } - } - }, - "version": "0.1", - "tags": [ - 4, - 5 - ], - "tools": [ - "filter", - "notebook-bp-validator", - "save" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "papermill_subworkflow", - "fields": { - "name": "Papermill", - "description": "Papermill is a tool for parameterizing and executing Jupyter Notebooks.", - "pipeline_step": "papermill_subworkflow:\r\n in:\r\n filter.regex: papermill_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#papermill_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n requirements:\r\n ScatterFeatureRequirement: {}\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: papermill\r\n filter.regex: string\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n []\r\n\r\n steps:\r\n filter_papermill_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n papermill_step:\r\n in:\r\n notebook_path: filter_papermill_step/file_list\r\n source_directory: source_directory\r\n scatter: notebook_path\r\n run: '#papermill_tool'\r\n out:\r\n - output_nb\r\n save_papermill_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: papermill_step/output_nb\r\n run_id: run_id\r\n server_url: server_url\r\n scatter: report\r\n run: '#save_tool'\r\n out: []\r\n id: papermill_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "filter": { - "regex": { - "type": "string", - "label": "regex", - "default": ".*\\.ipynb" - } - } - }, - "version": "0.1", - "tags": [ - 4, - 7 - ], - "tools": [ - "filter", - "papermill", - "save" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "pylint_subworkflow", - "fields": { - "name": "Pylint", - "description": "pylint - Static code analyser tool for Python", - "pipeline_step": "pylint_subworkflow:\r\n in:\r\n filter.regex: pylint_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n pylint.disable: pylint_subworkflow.pylint.disable\r\n pylint.errors_only: pylint_subworkflow.pylint.errors_only\r\n pylint.verbose: pylint_subworkflow.pylint.verbose\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#pylint_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: pylint\r\n filter.regex: string\r\n pipeline_id: string\r\n pylint.disable: string\r\n pylint.errors_only: boolean\r\n pylint.verbose: boolean\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n pylint_report:\r\n type: File\r\n outputSource: pylint_step/pylint_report\r\n\r\n steps:\r\n filter_pylint_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n pylint_step:\r\n in:\r\n disable: pylint.disable\r\n errors_only: pylint.errors_only\r\n file_list: filter_pylint_step/file_list\r\n source_directory: source_directory\r\n verbose: pylint.verbose\r\n run: '#pylint_tool'\r\n out:\r\n - pylint_report\r\n save_pylint_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: pylint_step/pylint_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: pylint_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "filter": { - "regex": { - "type": "string", - "label": "regex", - "default": ".*\\.py" - } - }, - "pylint": { - "disable": { - "doc": "Disable the message, report, category or checker with the given id(s).", - "type": "string", - "label": "Disable IDs", - "default": "E0401" - }, - "verbose": { - "doc": "In verbose mode, extra non-checker-related info will be displayed.", - "type": "boolean", - "label": "Verbose", - "default": false - }, - "errors_only": { - "doc": "In error mode, messages with a category besides ERROR or FATAL are suppressed, and no reports are done by default. Error mode is compatible with disabling specific errors.", - "type": "boolean", - "label": "Errors only", - "default": false - } - } - }, - "version": "0.1", - "tags": [ - 1, - 5 - ], - "tools": [ - "filter", - "pylint", - "save" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "ruff_ipynb_subworkflow", - "fields": { - "name": "Ruff - Notebook", - "description": "Ruff - An extremely fast Python linter and code formatter, written in Rust", - "pipeline_step": "ruff_ipynb_subworkflow:\r\n in:\r\n filter.regex: ruff_ipynb_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n ruff.verbose: ruff_ipynb_subworkflow.ruff.verbose\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#ruff_ipynb_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: ruff\r\n filter.regex: string\r\n pipeline_id: string\r\n ruff.verbose: boolean\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n ruff_report:\r\n type: File\r\n outputSource: ruff_step/ruff_report\r\n\r\n steps:\r\n filter_ruff_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n ruff_step:\r\n in:\r\n file_list: filter_ruff_step/file_list\r\n source_directory: source_directory\r\n verbose: ruff.verbose\r\n run: '#ruff_tool'\r\n out:\r\n - ruff_report\r\n save_ruff_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: ruff_step/ruff_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: ruff_ipynb_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "ruff": { - "verbose": { - "doc": "Enable verbose logging.", - "type": "boolean", - "label": "Verbose", - "default": false - } - }, - "filter": { - "regex": { - "type": "string", - "label": "regex", - "default": ".*\\.ipynb" - } - } - }, - "version": "0.1", - "tags": [ - 4, - 5 - ], - "tools": [ - "filter", - "ruff", - "save" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "ruff_subworkflow", - "fields": { - "name": "Ruff", - "description": "Ruff - An extremely fast Python linter and code formatter, written in Rust", - "pipeline_step": "ruff_subworkflow:\r\n in:\r\n filter.regex: ruff_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n ruff.verbose: ruff_subworkflow.ruff.verbose\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#ruff_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: ruff\r\n filter.regex: string\r\n pipeline_id: string\r\n ruff.verbose: boolean\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n ruff_report:\r\n type: File\r\n outputSource: ruff_step/ruff_report\r\n\r\n steps:\r\n filter_ruff_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n ruff_step:\r\n in:\r\n file_list: filter_ruff_step/file_list\r\n source_directory: source_directory\r\n verbose: ruff.verbose\r\n run: '#ruff_tool'\r\n out:\r\n - ruff_report\r\n save_ruff_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: ruff_step/ruff_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: ruff_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "ruff": { - "verbose": { - "doc": "Enable verbose logging.", - "type": "boolean", - "label": "Verbose", - "default": false - } - }, - "filter": { - "regex": { - "type": "string", - "label": "regex", - "default": ".*\\.py" - } - } - }, - "version": "0.1", - "tags": [ - 1, - 5 - ], - "tools": [ - "filter", - "ruff", - "save" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "sonarqube", - "fields": { - "name": "SonarQube", - "description": "SonarQube - Code Quality, Security & Static Analysis Tool\r\n\r\nThis tool creates a project in our internal SonarQube server, sends it the code for analysis, and then retrieves the analysis results for storage in the database.", - "pipeline_step": "sonarqube_workflow_step:\r\n in:\r\n pipeline_id: pipeline_id\r\n repo_path: clone_step/repo_directory\r\n run_id: run_id\r\n server_url: server_url\r\n sonarqube_project_key: sonarqube_project_key\r\n sonarqube_project_name: sonarqube_project_name\r\n sonarqube_server: sonarqube_server\r\n sonarqube_token: sonarqube_token\r\n run: '#sonarqube_workflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: sonarqube\r\n pipeline_id:\r\n type: string\r\n repo_path:\r\n type: Directory\r\n run_id:\r\n type: string\r\n server_url:\r\n type: string\r\n sonarqube_project_key:\r\n type: string\r\n sonarqube_project_name:\r\n type: string\r\n sonarqube_server:\r\n type: string\r\n default: sonarqube-sonarqube.sonarqube:9000\r\n sonarqube_token:\r\n type: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n save_sonarqube_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: sonarqube_get_report_step/sonarqube_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n sonarqube_create_project_step:\r\n in:\r\n sonarqube_project_key: sonarqube_project_key\r\n sonarqube_project_name: sonarqube_project_name\r\n sonarqube_server: sonarqube_server\r\n sonarqube_token: sonarqube_token\r\n run: '#sonarqube_create_project_tool'\r\n out:\r\n - sonarqube_project_key\r\n - sonarqube_server\r\n - sonarqube_token\r\n sonarqube_get_report_step:\r\n in:\r\n sonarqube_project_key: sonarqube_scan_step/sonarqube_project_key\r\n sonarqube_server: sonarqube_scan_step/sonarqube_server\r\n sonarqube_token: sonarqube_scan_step/sonarqube_token\r\n run: '#sonarqube_get_report_tool'\r\n out:\r\n - sonarqube_project_key\r\n - sonarqube_server\r\n - sonarqube_token\r\n - sonarqube_report\r\n sonarqube_scan_step:\r\n in:\r\n sonarqube_project_key: sonarqube_create_project_step/sonarqube_project_key\r\n sonarqube_server: sonarqube_create_project_step/sonarqube_server\r\n sonarqube_token: sonarqube_create_project_step/sonarqube_token\r\n source_directory: repo_path\r\n run: '#sonarqube_scan_tool'\r\n out:\r\n - sonarqube_project_key\r\n - sonarqube_server\r\n - sonarqube_token\r\n id: sonarqube_workflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": {}, - "version": "0.1", - "tags": [], - "tools": [ - "save", - "sonarqube_create_project", - "sonarqube_get_report", - "sonarqube_scan" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "trivy_subworkflow", - "fields": { - "name": "Trivy", - "description": "The all-in-one open source security scanner\r\nUse Trivy to find vulnerabilities (CVE) & misconfigurations (IaC) across code repositories, binary artifacts, container images, Kubernetes clusters, and more.", - "pipeline_step": "trivy_subworkflow:\r\n in:\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n trivy.image: trivy_subworkflow.trivy.image\r\n run: '#trivy_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: trivy\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n trivy.image: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n save_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: trivy_step/trivy_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n trivy_step:\r\n in:\r\n image: trivy.image\r\n run: '#trivy_tool'\r\n out:\r\n - trivy_report\r\n id: trivy_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "trivy": { - "image": { - "type": "string", - "label": "Docker image", - "default": "alpine/git" - } - } - }, - "version": "0.1", - "tags": [ - 6, - 9 - ], - "tools": [ - "save", - "trivy" - ] - } - }, - { - "model": "backend.commandlinetool", - "pk": "ap_validator", - "fields": { - "name": "Application Package Validator", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: nexus.spaceapplications.com/repository/docker-eoepca/ap_validator:2025-03-05.1\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n ap-validator \\\\\r\n --format json \\\\\r\n --detail '$(inputs.detail)' \\\\\r\n --entry-point '$(inputs.entry_point)' \\\\\r\n '$(inputs.file_path)' > ~/ap_validator_report.json\r\n\r\n exit 0\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n file_path: string\r\n source_directory: Directory\r\n detail: string\r\n entry_point: string\r\n\r\n outputs:\r\n ap_validator_report:\r\n type: File\r\n outputBinding:\r\n glob: ap_validator_report.json\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: ap_validator_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "bandit", - "fields": { - "name": "Bandit", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: cytopia/bandit\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n PARAMS=\"-f $(inputs.output_format) -o $HOME/$(inputs.output_file)\"\r\n if [ \"$(inputs.exit_zero)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS --exit-zero\"\r\n fi\r\n if [ \"$(inputs.verbose)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -v\"\r\n fi\r\n\r\n bandit $PARAMS $(inputs.file_list.join(\" \"))\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n exit_zero:\r\n label: Exit with zero\r\n doc: Exit with 0, even with results found.\r\n type: boolean\r\n default: true\r\n file_list: string[]\r\n output_file:\r\n label: Output file\r\n doc: Write report to filename.\r\n type: string\r\n default: bandit_report.json\r\n output_format:\r\n label: Output format\r\n doc: Specify output format.\r\n type: string\r\n default: json\r\n source_directory: Directory\r\n verbose: boolean\r\n\r\n outputs:\r\n bandit_report:\r\n type: File\r\n outputBinding:\r\n glob: $(inputs.output_file)\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: bandit_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "clone", - "fields": { - "name": "Clone repo", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: alpine/git\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: clone_branch.sh\r\n entry: |-\r\n set -e\r\n\r\n if [ $(inputs.repo_branch) ]; then\r\n echo 'Branch specified: $(inputs.repo_branch). Cloning branch...'\r\n git clone $(inputs.repo_url) -b $(inputs.repo_branch)\r\n else\r\n echo 'No branch specified. Cloning default branch...'\r\n git clone $(inputs.repo_url)\r\n fi\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n repo_branch: string\r\n repo_url: string\r\n\r\n outputs:\r\n repo_directory:\r\n type: Directory\r\n outputBinding:\r\n glob: $(inputs.repo_url.split('/').pop().replace('.git',''))\r\n\r\n baseCommand: sh\r\n arguments:\r\n - clone_branch.sh\r\n id: clone_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "filter", - "fields": { - "name": "Filter", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: alpine:latest\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n find . -type f -regex \"$(inputs.regex)\" > $HOME/filter.out\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n regex: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n file_list:\r\n type: string[]\r\n outputBinding:\r\n glob: filter.out\r\n outputEval: |-\r\n $(self[0].contents.split('\\n').filter(function(line) {return line.trim() !== '';}))\r\n loadContents: true\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: filter_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "flake8", - "fields": { - "name": "Flake8", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: eoepca/appquality-flake8-json:v0.1.0\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n PARAMS=\"--format=$(inputs.output_format) --output-file=$HOME/$(inputs.output_file)\"\r\n if [ \"$(inputs.exit_zero)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS --exit-zero\"\r\n fi\r\n if [ \"$(inputs.verbose)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -v\"\r\n fi\r\n\r\n flake8 $PARAMS $(inputs.file_list.join(\" \"))\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n exit_zero:\r\n label: Exit with zero\r\n doc: Force Flake8 to use the exit status code 0 even if there are errors.\r\n type: boolean\r\n default: true\r\n file_list: string[]\r\n output_file:\r\n label: Output file\r\n doc: Redirect all output to the specified file.\r\n type: string\r\n default: flake8_report.json\r\n output_format:\r\n label: Output format\r\n doc: Select the formatter used to display errors to the user.\r\n type: string\r\n default: json\r\n source_directory: Directory\r\n verbose: boolean\r\n\r\n outputs:\r\n flake8_report:\r\n type: File\r\n outputBinding:\r\n glob: $(inputs.output_file)\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: flake8_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "notebook-bp-validator", - "fields": { - "name": "Jupyter Notebook Best Practices Validator", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: eoepca/notebook-bp-validator:2025-05-12.1\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n PARAMS=\"-s $(inputs.schema)\"\r\n if [ '$(inputs.abspath)' == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -p\"\r\n fi\r\n\r\n nb-validator $PARAMS $(inputs.file_list.join(\" \")) > ~/notebook-bp-validator_report.json\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n abspath: boolean\r\n file_list: string[]\r\n schema: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n notebook-bp-validator_report:\r\n type: File\r\n outputBinding:\r\n glob: notebook-bp-validator_report.json\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: notebook-bp-validator_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "papermill", - "fields": { - "name": "Papermill", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: continuumio/miniconda3\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n set -e\r\n\r\n cat ~/err.ipynb > ~/out.ipynb\r\n cd $(inputs.source_directory.path)\r\n\r\n pip install pyyaml\r\n\r\n python ~/script.py $(inputs.notebook_path) | tee ~/environment.yml\r\n\r\n if [ ! -s ~/environment.yml ]; then\r\n echo \"Error: environment.yml is empty. Aborting environment creation.\"\r\n exit 1\r\n fi\r\n\r\n ENV=`grep \"name: \" ~/environment.yml | cut - -d' ' -f2`\r\n\r\n conda env create -f ~/environment.yml\r\n conda run -n $ENV pip install ipykernel papermill\r\n conda run -n $ENV ipython kernel install --user --name $ENV \r\n conda run -n $ENV papermill $(inputs.notebook_path) ~/out.ipynb\r\n - entryname: script.py\r\n entry: |-\r\n import json\r\n import yaml\r\n import sys\r\n\r\n try:\r\n with open('$(inputs.notebook_path)') as f:\r\n nb_json = json.load(f)\r\n except Exception as e:\r\n sys.stderr.write(f\"Failed to load notebook: {e}\\\\n\")\r\n sys.exit(1)\r\n\r\n try:\r\n env = nb_json[\"metadata\"][\"software_requirements\"][\"conda_environment\"]\r\n kernelspec = nb_json[\"metadata\"][\"kernelspec\"][\"name\"]\r\n except KeyError as e:\r\n sys.stderr.write(f\"Missing key in notebook metadata: {e}\\\\n\")\r\n sys.exit(1)\r\n\r\n if env[\"name\"] == kernelspec:\r\n if env.get(\"dependencies\"):\r\n res = yaml.dump({\"name\": env[\"name\"], \"dependencies\": env[\"dependencies\"]})\r\n print(res)\r\n else:\r\n sys.stderr.write(\"No dependencies found in the environment specification.\\\\n\")\r\n sys.exit(1)\r\n else:\r\n sys.stderr.write(\"Kernel name does not match the conda environment name.\\\\n\")\r\n sys.exit(1)\r\n - entryname: err.ipynb\r\n entry: |-\r\n {\r\n \"cells\": [\r\n {\r\n \"cell_type\": \"markdown\",\r\n \"metadata\": {},\r\n \"source\": [\r\n \"Error: The notebook could not be run.\"\r\n ]\r\n }\r\n ],\r\n \"metadata\": {\r\n \"language_info\": {\r\n \"name\": \"python\"\r\n }\r\n },\r\n \"nbformat\": 4,\r\n \"nbformat_minor\": 2\r\n }\r\n\r\n inputs:\r\n notebook_path: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n output_nb:\r\n type: File\r\n outputBinding:\r\n glob: out.ipynb\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: papermill_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "pylint", - "fields": { - "name": "Pylint", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: cytopia/pylint\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n PARAMS=\"--output-format=$(inputs.output_format) --output=$HOME/$(inputs.output_file) --disable=$(inputs.disable)\"\r\n if [ \"$(inputs.exit_zero)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS --exit-zero\"\r\n fi\r\n if [ \"$(inputs.errors_only)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -E\"\r\n fi\r\n if [ \"$(inputs.verbose)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -v\"\r\n fi\r\n\r\n pylint $PARAMS $(inputs.file_list.join(\" \"))\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n disable: string\r\n errors_only: boolean\r\n exit_zero:\r\n doc: |-\r\n Always return a 0 (non-error) status code, even if lint errors are found. This is primarily useful in continuous integration scripts.\r\n type: boolean\r\n default: true\r\n file_list: string[]\r\n output_file:\r\n doc: Specify an output file.\r\n type: string\r\n default: pylint_report.json\r\n output_format:\r\n doc: |-\r\n Set the output format. Available formats are: text, parseable, colorized, json2 (improved json format), json (old json format) and msvs (visual studio). You can also give a reporter class, e.g. mypackage.mymodule.MyReporterClass.\r\n type: string\r\n default: json\r\n source_directory: Directory\r\n verbose: boolean\r\n\r\n outputs:\r\n pylint_report:\r\n type: File\r\n outputBinding:\r\n glob: $(inputs.output_file)\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: pylint_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "ruff", - "fields": { - "name": "Ruff", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: ghcr.io/astral-sh/ruff:alpine\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n PARAMS=\"--output-format $(inputs.output_format) -o $HOME/$(inputs.output_file)\"\r\n if [ \"$(inputs.exit_zero)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -e\"\r\n fi\r\n if [ \"$(inputs.no_cache)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -n\"\r\n fi\r\n if [ \"$(inputs.verbose)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -v\"\r\n fi\r\n\r\n ruff check $PARAMS $(inputs.file_list.join(\" \"))\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n exit_zero:\r\n label: Exit with zero\r\n doc: Exit with status code \"0\", even upon detecting lint violations.\r\n type: boolean\r\n default: true\r\n no_cache:\r\n label: Disable cache\r\n doc: Disable cache reads.\r\n type: boolean\r\n default: true\r\n file_list: string[]\r\n output_file:\r\n label: Output file\r\n doc: Specify file to write the linter output to.\r\n type: string\r\n default: ruff_report.json\r\n output_format:\r\n label: Output format\r\n doc: |-\r\n Output serialization format for violations. Possible values: concise, full, json, json-lines, junit, grouped, github, gitlab, pylint, rdjson, azure, sarif.\r\n type: string\r\n default: json\r\n source_directory: Directory\r\n verbose: boolean\r\n\r\n outputs:\r\n ruff_report:\r\n type: File\r\n outputBinding:\r\n glob: $(inputs.output_file)\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: ruff_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "save", - "fields": { - "name": "Save report", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: curlimages/curl\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n name: string\r\n pipeline_id: string\r\n report: File\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n baseCommand: curl\r\n arguments:\r\n - prefix: -X\r\n valueFrom: POST\r\n - prefix: -L\r\n valueFrom: |-\r\n $(inputs.server_url + '/api/pipelines/' + inputs.pipeline_id + '/runs/' + inputs.run_id + '/jobreports/?name=' + inputs.name)\r\n - prefix: -H\r\n valueFrom: Content-Type:application/json\r\n - prefix: -d\r\n valueFrom: $('@' + inputs.report.path)\r\n id: save_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "sonarqube_create_project", - "fields": { - "name": "[SonarQube] Create project", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: curlimages/curl\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n sonarqube_project_key:\r\n type: string\r\n sonarqube_project_name:\r\n type: string\r\n sonarqube_server:\r\n type: string\r\n default: sonarqube-sonarqube.sonarqube:9000\r\n sonarqube_token:\r\n type: string\r\n\r\n outputs:\r\n sonarqube_project_key:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_project_key)\r\n sonarqube_server:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_server)\r\n sonarqube_token:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_token)\r\n\r\n baseCommand:\r\n - curl\r\n arguments:\r\n - prefix: -X\r\n valueFrom: POST\r\n - prefix: -L\r\n valueFrom: $('http://' + inputs.sonarqube_server + '/api/projects/create')\r\n - prefix: -u\r\n valueFrom: $(inputs.sonarqube_token + ':')\r\n - prefix: -d\r\n valueFrom: $('name=' + inputs.sonarqube_project_name)\r\n - prefix: -d\r\n valueFrom: $('project=' + inputs.sonarqube_project_key)\r\n id: sonarqube_create_project_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "sonarqube_get_report", - "fields": { - "name": "[SonarQube] Get report", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: curlimages/curl\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n sonarqube_project_key:\r\n type: string\r\n sonarqube_server:\r\n type: string\r\n default: sonarqube-sonarqube.sonarqube:9000\r\n sonarqube_token:\r\n type: string\r\n\r\n outputs:\r\n sonarqube_project_key:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_project_key)\r\n sonarqube_report:\r\n type: File\r\n outputBinding:\r\n glob: sonarqube_report.json\r\n sonarqube_server:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_server)\r\n sonarqube_token:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_token)\r\n stdout: sonarqube_report.json\r\n\r\n baseCommand:\r\n - curl\r\n arguments:\r\n - prefix: -L\r\n valueFrom: |-\r\n $('http://' + inputs.sonarqube_server + '/api/issues/search?components=' + inputs.sonarqube_project_key)\r\n - prefix: -u\r\n valueFrom: $(inputs.sonarqube_token + ':')\r\n id: sonarqube_get_report_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "sonarqube_scan", - "fields": { - "name": "[SonarQube] Scan repo", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: sonarsource/sonar-scanner-cli\r\n EnvVarRequirement:\r\n envDef:\r\n SONAR_HOST_URL: $('http://' + inputs.sonarqube_server)\r\n SONAR_TOKEN: $(inputs.sonarqube_token)\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n sonarqube_project_key:\r\n type: string\r\n sonarqube_server:\r\n type: string\r\n default: sonarqube-sonarqube.sonarqube:9000\r\n sonarqube_token:\r\n type: string\r\n source_directory:\r\n type: Directory\r\n\r\n outputs:\r\n sonarqube_project_key:\r\n type: string\r\n outputBinding:\r\n glob:\r\n outputEval: $(inputs.sonarqube_project_key)\r\n sonarqube_server:\r\n type: string\r\n outputBinding:\r\n glob:\r\n outputEval: $(inputs.sonarqube_server)\r\n sonarqube_token:\r\n type: string\r\n outputBinding:\r\n glob:\r\n outputEval: $(inputs.sonarqube_token)\r\n\r\n baseCommand:\r\n - sonar-scanner\r\n arguments:\r\n - prefix: -D\r\n valueFrom: $('sonar.projectKey=' + inputs.sonarqube_project_key)\r\n separate: false\r\n # - prefix: -D\r\n # valueFrom: $('sonar.userHome=$HOME')\r\n # separate: false\r\n - prefix: -D\r\n valueFrom: $('sonar.projectBaseDir=' + inputs.source_directory.path + '/../')\r\n separate: false\r\n # - prefix: -D\r\n # valueFrom: $('sonar.source=$HOME' /*+ inputs.source_directory.path*/)\r\n # separate: false\r\n - prefix: -X\r\n id: sonarqube_scan_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "trivy", - "fields": { - "name": "Trivy", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: aquasec/trivy:latest\r\n\r\n inputs:\r\n image: string\r\n\r\n outputs:\r\n trivy_report:\r\n type: File\r\n outputBinding:\r\n glob: trivy_report.json\r\n\r\n baseCommand: trivy\r\n arguments:\r\n - image\r\n - prefix: -f\r\n valueFrom: json\r\n - prefix: -o\r\n valueFrom: trivy_report.json\r\n - $(inputs.image)\r\n id: trivy_tool", - "version": "0.1" - } - } -] \ No newline at end of file diff --git a/backend/backend/fixtures/backend.json b/backend/backend/fixtures/backend.json deleted file mode 100644 index eb48f04..0000000 --- a/backend/backend/fixtures/backend.json +++ /dev/null @@ -1,716 +0,0 @@ -[ - { - "model": "backend.tag", - "pk": 1, - "fields": { - "name": "asset: python" - } - }, - { - "model": "backend.tag", - "pk": 2, - "fields": { - "name": "asset: other" - } - }, - { - "model": "backend.tag", - "pk": 3, - "fields": { - "name": "asset: cwl" - } - }, - { - "model": "backend.tag", - "pk": 4, - "fields": { - "name": "asset: notebook" - } - }, - { - "model": "backend.tag", - "pk": 5, - "fields": { - "name": "type: best practice" - } - }, - { - "model": "backend.tag", - "pk": 6, - "fields": { - "name": "type: app quality" - } - }, - { - "model": "backend.tag", - "pk": 7, - "fields": { - "name": "type: app performance" - } - }, - { - "model": "backend.tag", - "pk": 8, - "fields": { - "name": "type: init" - } - }, - { - "model": "backend.tag", - "pk": 9, - "fields": { - "name": "asset: docker" - } - }, - { - "model": "backend.commandlinetool", - "pk": "ap_validator", - "fields": { - "name": "Application Package Validator", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: nexus.spaceapplications.com/repository/docker-eoepca/ap_validator:2025-03-05.1\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n ap-validator \\\\\r\n --format json \\\\\r\n --detail '$(inputs.detail)' \\\\\r\n --entry-point '$(inputs.entry_point)' \\\\\r\n '$(inputs.file_path)' > ~/ap_validator_report.json\r\n\r\n exit 0\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n file_path: string\r\n source_directory: Directory\r\n detail: string\r\n entry_point: string\r\n\r\n outputs:\r\n ap_validator_report:\r\n type: File\r\n outputBinding:\r\n glob: ap_validator_report.json\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: ap_validator_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "bandit", - "fields": { - "name": "Bandit", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: cytopia/bandit\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n PARAMS=\"-f $(inputs.output_format) -o $HOME/$(inputs.output_file)\"\r\n if [ \"$(inputs.exit_zero)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS --exit-zero\"\r\n fi\r\n if [ \"$(inputs.verbose)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -v\"\r\n fi\r\n\r\n bandit $PARAMS $(inputs.file_list.join(\" \"))\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n exit_zero:\r\n label: Exit with zero\r\n doc: Exit with 0, even with results found.\r\n type: boolean\r\n default: true\r\n file_list: string[]\r\n output_file:\r\n label: Output file\r\n doc: Write report to filename.\r\n type: string\r\n default: bandit_report.json\r\n output_format:\r\n label: Output format\r\n doc: Specify output format.\r\n type: string\r\n default: json\r\n source_directory: Directory\r\n verbose: boolean\r\n\r\n outputs:\r\n bandit_report:\r\n type: File\r\n outputBinding:\r\n glob: $(inputs.output_file)\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: bandit_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "clone", - "fields": { - "name": "Clone repo", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: alpine/git\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: clone_branch.sh\r\n entry: |-\r\n set -e\r\n\r\n if [ $(inputs.repo_branch) ]; then\r\n echo 'Branch specified: $(inputs.repo_branch). Cloning branch...'\r\n git clone $(inputs.repo_url) -b $(inputs.repo_branch)\r\n else\r\n echo 'No branch specified. Cloning default branch...'\r\n git clone $(inputs.repo_url)\r\n fi\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n repo_branch: string\r\n repo_url: string\r\n\r\n outputs:\r\n repo_directory:\r\n type: Directory\r\n outputBinding:\r\n glob: $(inputs.repo_url.split('/').pop().replace('.git',''))\r\n\r\n baseCommand: sh\r\n arguments:\r\n - clone_branch.sh\r\n id: clone_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "filter", - "fields": { - "name": "Filter", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: alpine:latest\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n find . -type f -regex \"$(inputs.regex)\" > $HOME/filter.out\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n regex: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n file_list:\r\n type: string[]\r\n outputBinding:\r\n glob: filter.out\r\n outputEval: |-\r\n $(self[0].contents.split('\\n').filter(function(line) {return line.trim() !== '';}))\r\n loadContents: true\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: filter_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "flake8", - "fields": { - "name": "Flake8", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: eoepca/appquality-flake8-json:v0.1.0\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n PARAMS=\"--format=$(inputs.output_format) --output-file=$HOME/$(inputs.output_file)\"\r\n if [ \"$(inputs.exit_zero)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS --exit-zero\"\r\n fi\r\n if [ \"$(inputs.verbose)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -v\"\r\n fi\r\n\r\n flake8 $PARAMS $(inputs.file_list.join(\" \"))\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n exit_zero:\r\n label: Exit with zero\r\n doc: Force Flake8 to use the exit status code 0 even if there are errors.\r\n type: boolean\r\n default: true\r\n file_list: string[]\r\n output_file:\r\n label: Output file\r\n doc: Redirect all output to the specified file.\r\n type: string\r\n default: flake8_report.json\r\n output_format:\r\n label: Output format\r\n doc: Select the formatter used to display errors to the user.\r\n type: string\r\n default: json\r\n source_directory: Directory\r\n verbose: boolean\r\n\r\n outputs:\r\n flake8_report:\r\n type: File\r\n outputBinding:\r\n glob: $(inputs.output_file)\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: flake8_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "notebook-bp-validator", - "fields": { - "name": "Jupyter Notebook Best Practices Validator", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: eoepca/notebook-bp-validator:2025-05-12.1\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n PARAMS=\"-s $(inputs.schema)\"\r\n if [ '$(inputs.abspath)' == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -p\"\r\n fi\r\n\r\n nb-validator $PARAMS $(inputs.file_list.join(\" \")) > ~/notebook-bp-validator_report.json\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n abspath: boolean\r\n file_list: string[]\r\n schema: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n notebook-bp-validator_report:\r\n type: File\r\n outputBinding:\r\n glob: notebook-bp-validator_report.json\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: notebook-bp-validator_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "papermill", - "fields": { - "name": "Papermill", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: continuumio/miniconda3\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n set -e\r\n\r\n cat ~/err.ipynb > ~/out.ipynb\r\n echo \"[Papermill] Source directory: $(inputs.source_directory.path)\"\r\n echo \"[Papermill] Notebook path: $(inputs.notebook_path)\"\r\n echo \"[Papermill] Extract requirements: $(inputs.extract_requirements)\"\r\n echo \"[Papermill] Extra requirements: $(inputs.extra_requirements)\"\r\n echo \"[Papermill] Execution parameters: $(inputs.execution_parameters)\"\r\n\r\n cd $(inputs.source_directory.path)\r\n\r\n if [ -s environment.yml ]; then\r\n echo \"Info: Found non-empty environment.yml file in the source folder. Will use it to setup the Conda environment.\"\r\n ENV=`grep \"name: \" environment.yml | cut - -d' ' -f2`\r\n echo \"Info: Environment name: $ENV\"\r\n cp -p environment.yml ~/environment.yml\r\n touch ~/requirements.txt\r\n else\r\n if [ \"$(inputs.extract_requirements)\" = \"true\" ]; then\r\n echo \"Info: Generating requirements.txt\"\r\n cd ~\r\n apt update && apt install -y jq\r\n pip install pipreqsnb\r\n pipreqsnb --force --encoding utf-8 --savepath ~/requirements.txt $(inputs.source_directory.path)/$(inputs.notebook_path)\r\n echo \"Info: Generating environment.yml\"\r\n ENV=`cat $(inputs.source_directory.path)/$(inputs.notebook_path) | jq .metadata.kernelspec.name -r`\r\n echo \"Info: Environment name: $ENV\"\r\n echo \"name: $ENV\" > ~/environment.yml\r\n else\r\n echo \"Info: Generating environment.yml\"\r\n pip install pyyaml\r\n python ~/script.py $(inputs.notebook_path) | tee ~/environment.yml\r\n touch ~/requirements.txt\r\n fi\r\n fi\r\n\r\n if [ ! -s ~/environment.yml ]; then\r\n echo \"Error: environment.yml is empty. Aborting environment creation.\"\r\n exit 1\r\n fi\r\n\r\n echo \"Requirements file:\"\r\n cat ~/requirements.txt\r\n\r\n echo \"Environment file:\"\r\n cat ~/environment.yml\r\n\r\n ENV=`grep \"name: \" ~/environment.yml | cut - -d' ' -f2`\r\n echo \"Info: Environment name: $ENV\"\r\n\r\n conda env create -f ~/environment.yml\r\n conda run -n $ENV pip install ipykernel papermill\r\n conda run -n $ENV pip install -r ~/requirements.txt\r\n if [ \"$(inputs.extra_requirements)\" != \"\" ]; then\r\n conda run -n $ENV pip install $(inputs.extra_requirements)\r\n fi\r\n conda run -n $ENV ipython kernel install --user --name $ENV\r\n conda run -n $ENV papermill $(inputs.source_directory.path)/$(inputs.notebook_path) ~/out.ipynb\r\n\r\n - entryname: script.py\r\n entry: |-\r\n import json\r\n import yaml\r\n import sys\r\n\r\n try:\r\n with open('$(inputs.notebook_path)') as f:\r\n nb_json = json.load(f)\r\n except Exception as e:\r\n sys.stderr.write(f\"Failed to load notebook: {e}\\n\")\r\n sys.exit(1)\r\n\r\n conda_env = nb_json.get(\"metadata\", {}).get(\"software_requirements\", {}).get(\"conda_environment\", None)\r\n\r\n try:\r\n kernel_name = nb_json[\"metadata\"][\"kernelspec\"][\"name\"]\r\n except KeyError as e:\r\n sys.stderr.write(f\"Missing key in notebook metadata: {e}\\n\")\r\n sys.exit(1)\r\n\r\n if conda_env:\r\n if conda_env.get(\"name\", None) == kernel_name:\r\n if conda_env.get(\"dependencies\"):\r\n res = yaml.dump({\"name\": conda_env[\"name\"], \"dependencies\": conda_env[\"dependencies\"]})\r\n print(res)\r\n else:\r\n sys.stderr.write(\"No dependencies found in the environment specification.\\n\")\r\n sys.exit(1)\r\n else:\r\n sys.stderr.write(\"Kernel name does not match the conda environment name.\\n\")\r\n sys.exit(1)\r\n else:\r\n res = yaml.dump({\"name\": kernel_name, \"dependencies\": []})\r\n print(res)\r\n\r\n - entryname: err.ipynb\r\n entry: |-\r\n {\r\n \"cells\": [\r\n {\r\n \"cell_type\": \"markdown\",\r\n \"metadata\": {},\r\n \"source\": [\r\n \"Error: The notebook could not be run.\"\r\n ]\r\n }\r\n ],\r\n \"metadata\": {\r\n \"language_info\": {\r\n \"name\": \"python\"\r\n }\r\n },\r\n \"nbformat\": 4,\r\n \"nbformat_minor\": 2\r\n }\r\n\r\n inputs:\r\n notebook_path: string\r\n extra_requirements:\r\n label: Python requirements\r\n doc: Space-separated list of Python libraries to be installed before executing the notebook\r\n type: string\r\n default: \"\"\r\n extract_requirements:\r\n label: Extract requirements\r\n doc: Inspect the notebook cells to derive library requirements\r\n type: boolean\r\n default: false\r\n execution_parameters:\r\n label: Execution parameters\r\n doc: These parameters are provided as inputs to the executed notebooks\r\n type: string\r\n default: \"\"\r\n source_directory: Directory\r\n\r\n outputs:\r\n output_nb:\r\n type: File\r\n outputBinding:\r\n glob: out.ipynb\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: papermill_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "pylint", - "fields": { - "name": "Pylint", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: cytopia/pylint\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n PARAMS=\"--output-format=$(inputs.output_format) --output=$HOME/$(inputs.output_file) --disable=$(inputs.disable)\"\r\n if [ \"$(inputs.exit_zero)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS --exit-zero\"\r\n fi\r\n if [ \"$(inputs.errors_only)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -E\"\r\n fi\r\n if [ \"$(inputs.verbose)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -v\"\r\n fi\r\n\r\n pylint $PARAMS $(inputs.file_list.join(\" \"))\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n disable: string\r\n errors_only: boolean\r\n exit_zero:\r\n doc: |-\r\n Always return a 0 (non-error) status code, even if lint errors are found. This is primarily useful in continuous integration scripts.\r\n type: boolean\r\n default: true\r\n file_list: string[]\r\n output_file:\r\n doc: Specify an output file.\r\n type: string\r\n default: pylint_report.json\r\n output_format:\r\n doc: |-\r\n Set the output format. Available formats are: text, parseable, colorized, json2 (improved json format), json (old json format) and msvs (visual studio). You can also give a reporter class, e.g. mypackage.mymodule.MyReporterClass.\r\n type: string\r\n default: json\r\n source_directory: Directory\r\n verbose: boolean\r\n\r\n outputs:\r\n pylint_report:\r\n type: File\r\n outputBinding:\r\n glob: $(inputs.output_file)\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: pylint_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "ruff", - "fields": { - "name": "Ruff", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: ghcr.io/astral-sh/ruff:alpine\r\n InitialWorkDirRequirement:\r\n listing:\r\n - entryname: script.sh\r\n entry: |-\r\n cd $(inputs.source_directory.path)\r\n\r\n PARAMS=\"--output-format $(inputs.output_format) -o $HOME/$(inputs.output_file)\"\r\n if [ \"$(inputs.exit_zero)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -e\"\r\n fi\r\n if [ \"$(inputs.no_cache)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -n\"\r\n fi\r\n if [ \"$(inputs.verbose)\" == \"true\" ] ; then\r\n PARAMS=\"$PARAMS -v\"\r\n fi\r\n\r\n ruff check $PARAMS $(inputs.file_list.join(\" \"))\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n exit_zero:\r\n label: Exit with zero\r\n doc: Exit with status code \"0\", even upon detecting lint violations.\r\n type: boolean\r\n default: true\r\n no_cache:\r\n label: Disable cache\r\n doc: Disable cache reads.\r\n type: boolean\r\n default: true\r\n file_list: string[]\r\n output_file:\r\n label: Output file\r\n doc: Specify file to write the linter output to.\r\n type: string\r\n default: ruff_report.json\r\n output_format:\r\n label: Output format\r\n doc: |-\r\n Output serialization format for violations. Possible values: concise, full, json, json-lines, junit, grouped, github, gitlab, pylint, rdjson, azure, sarif.\r\n type: string\r\n default: json\r\n source_directory: Directory\r\n verbose: boolean\r\n\r\n outputs:\r\n ruff_report:\r\n type: File\r\n outputBinding:\r\n glob: $(inputs.output_file)\r\n\r\n baseCommand: sh\r\n arguments:\r\n - script.sh\r\n id: ruff_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "save", - "fields": { - "name": "Save report", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: curlimages/curl\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n name: string\r\n instance:\r\n type: string\r\n default: \"\"\r\n pipeline_id: string\r\n report: File\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n baseCommand: curl\r\n arguments:\r\n - prefix: -X\r\n valueFrom: POST\r\n - prefix: -L\r\n valueFrom: |-\r\n $(inputs.server_url + '/api/pipelines/' + inputs.pipeline_id + '/runs/' + inputs.run_id + '/jobreports/?name=' + inputs.name + '&instance=' + inputs.instance)\r\n - prefix: -H\r\n valueFrom: Content-Type:application/json\r\n - prefix: -d\r\n valueFrom: $('@' + inputs.report.path)\r\n id: save_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "sonarqube_create_project", - "fields": { - "name": "[SonarQube] Create project", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: curlimages/curl\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n sonarqube_project_key:\r\n type: string\r\n sonarqube_project_name:\r\n type: string\r\n sonarqube_server:\r\n type: string\r\n default: sonarqube-sonarqube.sonarqube:9000\r\n sonarqube_token:\r\n type: string\r\n\r\n outputs:\r\n sonarqube_project_key:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_project_key)\r\n sonarqube_server:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_server)\r\n sonarqube_token:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_token)\r\n\r\n baseCommand:\r\n - curl\r\n arguments:\r\n - prefix: -X\r\n valueFrom: POST\r\n - prefix: -L\r\n valueFrom: $('http://' + inputs.sonarqube_server + '/api/projects/create')\r\n - prefix: -u\r\n valueFrom: $(inputs.sonarqube_token + ':')\r\n - prefix: -d\r\n valueFrom: $('name=' + inputs.sonarqube_project_name)\r\n - prefix: -d\r\n valueFrom: $('project=' + inputs.sonarqube_project_key)\r\n id: sonarqube_create_project_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "sonarqube_get_report", - "fields": { - "name": "[SonarQube] Get report", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: curlimages/curl\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n sonarqube_project_key:\r\n type: string\r\n sonarqube_server:\r\n type: string\r\n default: sonarqube-sonarqube.sonarqube:9000\r\n sonarqube_token:\r\n type: string\r\n\r\n outputs:\r\n sonarqube_project_key:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_project_key)\r\n sonarqube_report:\r\n type: File\r\n outputBinding:\r\n glob: sonarqube_report.json\r\n sonarqube_server:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_server)\r\n sonarqube_token:\r\n type: string\r\n outputBinding:\r\n outputEval: $(inputs.sonarqube_token)\r\n stdout: sonarqube_report.json\r\n\r\n baseCommand:\r\n - curl\r\n arguments:\r\n - prefix: -L\r\n valueFrom: |-\r\n $('http://' + inputs.sonarqube_server + '/api/issues/search?components=' + inputs.sonarqube_project_key)\r\n - prefix: -u\r\n valueFrom: $(inputs.sonarqube_token + ':')\r\n id: sonarqube_get_report_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "sonarqube_scan", - "fields": { - "name": "[SonarQube] Scan repo", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: sonarsource/sonar-scanner-cli\r\n EnvVarRequirement:\r\n envDef:\r\n SONAR_HOST_URL: $('http://' + inputs.sonarqube_server)\r\n SONAR_TOKEN: $(inputs.sonarqube_token)\r\n InlineJavascriptRequirement: {}\r\n\r\n inputs:\r\n sonarqube_project_key:\r\n type: string\r\n sonarqube_server:\r\n type: string\r\n default: sonarqube-sonarqube.sonarqube:9000\r\n sonarqube_token:\r\n type: string\r\n source_directory:\r\n type: Directory\r\n\r\n outputs:\r\n sonarqube_project_key:\r\n type: string\r\n outputBinding:\r\n glob:\r\n outputEval: $(inputs.sonarqube_project_key)\r\n sonarqube_server:\r\n type: string\r\n outputBinding:\r\n glob:\r\n outputEval: $(inputs.sonarqube_server)\r\n sonarqube_token:\r\n type: string\r\n outputBinding:\r\n glob:\r\n outputEval: $(inputs.sonarqube_token)\r\n\r\n baseCommand:\r\n - sonar-scanner\r\n arguments:\r\n - prefix: -D\r\n valueFrom: $('sonar.projectKey=' + inputs.sonarqube_project_key)\r\n separate: false\r\n # - prefix: -D\r\n # valueFrom: $('sonar.userHome=$HOME')\r\n # separate: false\r\n - prefix: -D\r\n valueFrom: $('sonar.projectBaseDir=' + inputs.source_directory.path + '/../')\r\n separate: false\r\n # - prefix: -D\r\n # valueFrom: $('sonar.source=$HOME' /*+ inputs.source_directory.path*/)\r\n # separate: false\r\n - prefix: -X\r\n id: sonarqube_scan_tool", - "version": "0.1" - } - }, - { - "model": "backend.commandlinetool", - "pk": "trivy", - "fields": { - "name": "Trivy", - "definition": "- class: CommandLineTool\r\n\r\n requirements:\r\n DockerRequirement:\r\n dockerPull: aquasec/trivy:latest\r\n\r\n inputs:\r\n image: string\r\n\r\n outputs:\r\n trivy_report:\r\n type: File\r\n outputBinding:\r\n glob: trivy_report.json\r\n\r\n baseCommand: trivy\r\n arguments:\r\n - image\r\n - prefix: -f\r\n valueFrom: json\r\n - prefix: -o\r\n valueFrom: trivy_report.json\r\n - $(inputs.image)\r\n id: trivy_tool", - "version": "0.1" - } - }, - { - "model": "backend.subworkflow", - "pk": "ap_validator_subworkflow", - "fields": { - "name": "Application Package Validator", - "description": "Validation tool for checking OGC compliance of CWL files for application packages.", - "pipeline_step": "ap_validator_subworkflow:\r\n in:\r\n ap_validator.detail: ap_validator_subworkflow.ap_validator.detail\r\n ap_validator.entry_point: ap_validator_subworkflow.ap_validator.entry_point\r\n filter.regex: ap_validator_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#ap_validator_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n requirements:\r\n ScatterFeatureRequirement: {}\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: ap_validator\r\n ap_validator.detail: string\r\n ap_validator.entry_point: string\r\n filter.regex: string\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n []\r\n\r\n steps:\r\n filter_ap_validator_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n ap_validator_step:\r\n in:\r\n file_path: filter_ap_validator_step/file_list\r\n source_directory: source_directory\r\n detail: ap_validator.detail\r\n entry_point: ap_validator.entry_point\r\n scatter: file_path\r\n run: '#ap_validator_tool'\r\n out:\r\n - ap_validator_report\r\n save_ap_validator_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: ap_validator_step/ap_validator_report\r\n run_id: run_id\r\n server_url: server_url\r\n scatter: report\r\n run: '#save_tool'\r\n out: []\r\n id: ap_validator_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "filter": { - "regex": { - "type": "string", - "label": "regex", - "default": ".*\\.cwl" - } - }, - "ap_validator": { - "detail": { - "doc": "Output detail (none|errors|hints|all). Default: hints", - "type": "string", - "label": "Detail", - "default": "hints" - }, - "entry_point": { - "doc": "Name of entry point (Workflow or CommandLineTool)", - "type": "string", - "label": "Entry point", - "default": "main" - } - } - }, - "version": "0.1", - "tags": [ - 3, - 5 - ], - "tools": [ - "filter", - "ap_validator", - "save" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "bandit_subworkflow", - "fields": { - "name": "Bandit", - "description": "Bandit - Bandit is a tool designed to find common security issues in Python code", - "pipeline_step": "bandit_subworkflow:\r\n in:\r\n bandit.verbose: bandit_subworkflow.bandit.verbose\r\n filter.regex: bandit_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#bandit_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: bandit\r\n bandit.verbose: boolean\r\n filter.regex: string\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n bandit_report:\r\n type: File\r\n outputSource: bandit_step/bandit_report\r\n\r\n steps:\r\n bandit_step:\r\n in:\r\n file_list: filter_bandit_step/file_list\r\n source_directory: source_directory\r\n verbose: bandit.verbose\r\n run: '#bandit_tool'\r\n out:\r\n - bandit_report\r\n filter_bandit_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n save_bandit_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: bandit_step/bandit_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: bandit_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "bandit": { - "verbose": { - "doc": "Output extra information like excluded and included files.", - "type": "boolean", - "label": "Verbose", - "default": false - } - }, - "filter": { - "regex": { - "type": "string", - "label": "regex", - "default": ".*\\.py" - } - } - }, - "version": "0.1", - "tags": [ - 1, - 6 - ], - "tools": [ - "filter", - "bandit", - "save" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "clone_subworkflow", - "fields": { - "name": "Clone repo", - "description": "git-clone - Clone a repository into a new directory", - "pipeline_step": "clone_step:\r\n in:\r\n clone.repo_branch: clone_subworkflow.clone.repo_branch\r\n clone.repo_url: clone_subworkflow.clone.repo_url\r\n run: '#clone_subworkflow'\r\n out:\r\n - repo_directory", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n clone.repo_branch:\r\n type: string\r\n default: ''\r\n clone.repo_url: string\r\n\r\n outputs:\r\n repo_directory:\r\n type: Directory\r\n outputSource: clone_tool_step/repo_directory\r\n\r\n steps:\r\n clone_tool_step:\r\n in:\r\n repo_branch: clone.repo_branch\r\n repo_url: clone.repo_url\r\n run: '#clone_tool'\r\n out:\r\n - repo_directory\r\n id: clone_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "clone": { - "repo_url": { - "doc": "URL to the repository to clone.", - "type": "string", - "label": "Repo URL", - "default": "" - }, - "repo_branch": { - "doc": "Branch to checkout instead of the remote HEAD.", - "type": "string", - "label": "Repo branch", - "default": "" - } - } - }, - "version": "0.1", - "tags": [ - 2, - 8 - ], - "tools": [ - "clone" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "flake8_subworkflow", - "fields": { - "name": "Flake8", - "description": "flake8 - Style guide enforcement tool for Python", - "pipeline_step": "flake8_subworkflow:\r\n in:\r\n filter.regex: flake8_subworkflow.filter.regex\r\n flake8.verbose: flake8_subworkflow.flake8.verbose\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#flake8_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: flake8\r\n filter.regex: string\r\n flake8.verbose: boolean\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n flake8_report:\r\n type: File\r\n outputSource: flake8_step/flake8_report\r\n\r\n steps:\r\n filter_flake8_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n flake8_step:\r\n in:\r\n file_list: filter_flake8_step/file_list\r\n source_directory: source_directory\r\n verbose: flake8.verbose\r\n run: '#flake8_tool'\r\n out:\r\n - flake8_report\r\n save_flake8_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: flake8_step/flake8_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: flake8_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "filter": { - "regex": { - "type": "string", - "label": "regex", - "default": ".*\\.py" - } - }, - "flake8": { - "verbose": { - "doc": "Increase the verbosity of Flake8’s output.", - "type": "boolean", - "label": "Verbose", - "default": false - } - } - }, - "version": "0.1", - "tags": [ - 1, - 5 - ], - "tools": [ - "filter", - "flake8", - "save" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "notebook-bp-validator_subworkflow", - "fields": { - "name": "Jupyter Notebook Best Practices Validator", - "description": "This tool aims at validating the notebooks against the CEOS Jupyter Notebook Best Practice v1.1 document.", - "pipeline_step": "notebook-bp-validator_subworkflow:\r\n in:\r\n filter.regex: notebook-bp-validator_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n notebook-bp-validator.abspath: notebook-bp-validator_subworkflow.notebook-bp-validator.abspath\r\n notebook-bp-validator.schema: notebook-bp-validator_subworkflow.notebook-bp-validator.schema\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#notebook-bp-validator_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: notebook-bp-validator\r\n filter.regex: string\r\n pipeline_id: string\r\n notebook-bp-validator.abspath: boolean\r\n notebook-bp-validator.schema: string\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n notebook-bp-validator_report:\r\n type: File\r\n outputSource: notebook-bp-validator_step/notebook-bp-validator_report\r\n\r\n steps:\r\n filter_notebook-bp-validator_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n notebook-bp-validator_step:\r\n in:\r\n abspath: notebook-bp-validator.abspath \r\n file_list: filter_notebook-bp-validator_step/file_list\r\n source_directory: source_directory\r\n schema: notebook-bp-validator.schema\r\n run: '#notebook-bp-validator_tool'\r\n out:\r\n - notebook-bp-validator_report\r\n save_notebook-bp-validator_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: notebook-bp-validator_step/notebook-bp-validator_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: notebook-bp-validator_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "filter": { - "regex": { - "type": "string", - "label": "regex", - "default": ".*\\.ipynb" - } - }, - "notebook-bp-validator": { - "schema": { - "doc": "Supported values: 'eumetsat' or 'schema.org'", - "type": "string", - "label": "Schema", - "default": "eumetsat" - }, - "abspath": { - "doc": "Uses absolute paths in output.", - "type": "boolean", - "label": "Absolute path", - "default": false - } - } - }, - "version": "0.1", - "tags": [ - 4, - 5 - ], - "tools": [ - "filter", - "notebook-bp-validator", - "save" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "papermill_subworkflow", - "fields": { - "name": "Papermill", - "description": "Papermill is a tool for parameterizing and executing Jupyter Notebooks.", - "pipeline_step": "papermill_subworkflow:\r\n in:\r\n filter.regex: papermill_subworkflow.filter.regex\r\n papermill.extract_requirements: papermill_subworkflow.papermill.extract_requirements\r\n papermill.extra_requirements: papermill_subworkflow.papermill.extra_requirements\r\n papermill.execution_parameters: papermill_subworkflow.papermill.execution_parameters\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#papermill_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n # --- This workflow runs Papermill on a single notebook and saves the result.\r\n\r\n id: papermill_execution_subworkflow\r\n\r\n inputs:\r\n notebook_path: string # Single notebook file\r\n\r\n # Inputs for papermill_tool\r\n source_directory: Directory\r\n extract_requirements: boolean?\r\n extra_requirements: string?\r\n execution_parameters: string?\r\n\r\n # Inputs for save_tool\r\n name: string\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n papermill_step:\r\n in:\r\n notebook_path: notebook_path\r\n extract_requirements: extract_requirements\r\n extra_requirements: extra_requirements\r\n execution_parameters: execution_parameters\r\n source_directory: source_directory\r\n run: '#papermill_tool'\r\n out:\r\n - output_nb\r\n\r\n save_papermill_step:\r\n in:\r\n name: name\r\n instance: notebook_path\r\n pipeline_id: pipeline_id\r\n report: papermill_step/output_nb\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n\r\n- class: Workflow\r\n\r\n # --- This workflow filters the input files then executes the above sub-workflow on each of them (scatter)\r\n\r\n id: papermill_subworkflow\r\n\r\n requirements:\r\n ScatterFeatureRequirement: {}\r\n\r\n inputs:\r\n filter.regex: string\r\n papermill.extract_requirements: boolean\r\n papermill.extra_requirements: string\r\n papermill.execution_parameters: string\r\n name:\r\n type: string\r\n default: papermill\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs: []\r\n\r\n steps:\r\n filter_papermill_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list # Array of strings (the input for scatter)\r\n\r\n execute_notebooks_step:\r\n # Run the subworkflow defined above\r\n run: '#papermill_execution_subworkflow'\r\n # Scatter the above sub-workflow [Papermill => Save] over the array of files\r\n scatter: notebook_path\r\n in:\r\n # The list of notebooks filtered in the previous step\r\n notebook_path: filter_papermill_step/file_list\r\n # Other sub-workflow inputs\r\n source_directory: source_directory\r\n extract_requirements: papermill.extract_requirements\r\n extra_requirements: papermill.extra_requirements\r\n execution_parameters: papermill.execution_parameters\r\n name: name\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n\r\n out: []\r\n\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "filter": { - "regex": { - "type": "string", - "label": "regex", - "default": ".*\\.ipynb" - } - }, - "papermill": { - "extra_requirements": { - "doc": "Python requirements", - "type": "string", - "label": "Python requirements", - "default": "" - }, - "execution_parameters": { - "doc": "E.g. alpha=0.6, ratio=0.1", - "type": "string", - "label": "Execution parameters", - "default": "" - }, - "extract_requirements": { - "doc": "Extract requirements", - "type": "boolean", - "label": "Extract requirements", - "default": false - } - } - }, - "version": "0.1", - "tags": [ - 4, - 7 - ], - "tools": [ - "filter", - "papermill", - "save" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "pylint_subworkflow", - "fields": { - "name": "Pylint", - "description": "pylint - Static code analyser tool for Python", - "pipeline_step": "pylint_subworkflow:\r\n in:\r\n filter.regex: pylint_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n pylint.disable: pylint_subworkflow.pylint.disable\r\n pylint.errors_only: pylint_subworkflow.pylint.errors_only\r\n pylint.verbose: pylint_subworkflow.pylint.verbose\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#pylint_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: pylint\r\n filter.regex: string\r\n pipeline_id: string\r\n pylint.disable: string\r\n pylint.errors_only: boolean\r\n pylint.verbose: boolean\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n pylint_report:\r\n type: File\r\n outputSource: pylint_step/pylint_report\r\n\r\n steps:\r\n filter_pylint_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n pylint_step:\r\n in:\r\n disable: pylint.disable\r\n errors_only: pylint.errors_only\r\n file_list: filter_pylint_step/file_list\r\n source_directory: source_directory\r\n verbose: pylint.verbose\r\n run: '#pylint_tool'\r\n out:\r\n - pylint_report\r\n save_pylint_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: pylint_step/pylint_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: pylint_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "filter": { - "regex": { - "type": "string", - "label": "regex", - "default": ".*\\.py" - } - }, - "pylint": { - "disable": { - "doc": "Disable the message, report, category or checker with the given id(s).", - "type": "string", - "label": "Disable IDs", - "default": "E0401" - }, - "verbose": { - "doc": "In verbose mode, extra non-checker-related info will be displayed.", - "type": "boolean", - "label": "Verbose", - "default": false - }, - "errors_only": { - "doc": "In error mode, messages with a category besides ERROR or FATAL are suppressed, and no reports are done by default. Error mode is compatible with disabling specific errors.", - "type": "boolean", - "label": "Errors only", - "default": false - } - } - }, - "version": "0.1", - "tags": [ - 1, - 5 - ], - "tools": [ - "filter", - "pylint", - "save" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "ruff_ipynb_subworkflow", - "fields": { - "name": "Ruff - Notebook", - "description": "Ruff - An extremely fast Python linter and code formatter, written in Rust", - "pipeline_step": "ruff_ipynb_subworkflow:\r\n in:\r\n filter.regex: ruff_ipynb_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n ruff.verbose: ruff_ipynb_subworkflow.ruff.verbose\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#ruff_ipynb_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: ruff\r\n filter.regex: string\r\n pipeline_id: string\r\n ruff.verbose: boolean\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n ruff_report:\r\n type: File\r\n outputSource: ruff_step/ruff_report\r\n\r\n steps:\r\n filter_ruff_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n ruff_step:\r\n in:\r\n file_list: filter_ruff_step/file_list\r\n source_directory: source_directory\r\n verbose: ruff.verbose\r\n run: '#ruff_tool'\r\n out:\r\n - ruff_report\r\n save_ruff_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: ruff_step/ruff_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: ruff_ipynb_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "ruff": { - "verbose": { - "doc": "Enable verbose logging.", - "type": "boolean", - "label": "Verbose", - "default": false - } - }, - "filter": { - "regex": { - "type": "string", - "label": "regex", - "default": ".*\\.ipynb" - } - } - }, - "version": "0.1", - "tags": [ - 4, - 5 - ], - "tools": [ - "filter", - "ruff", - "save" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "ruff_subworkflow", - "fields": { - "name": "Ruff", - "description": "Ruff - An extremely fast Python linter and code formatter, written in Rust", - "pipeline_step": "ruff_subworkflow:\r\n in:\r\n filter.regex: ruff_subworkflow.filter.regex\r\n pipeline_id: pipeline_id\r\n ruff.verbose: ruff_subworkflow.ruff.verbose\r\n run_id: run_id\r\n server_url: server_url\r\n source_directory: clone_step/repo_directory\r\n run: '#ruff_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: ruff\r\n filter.regex: string\r\n pipeline_id: string\r\n ruff.verbose: boolean\r\n run_id: string\r\n server_url: string\r\n source_directory: Directory\r\n\r\n outputs:\r\n ruff_report:\r\n type: File\r\n outputSource: ruff_step/ruff_report\r\n\r\n steps:\r\n filter_ruff_step:\r\n in:\r\n regex: filter.regex\r\n source_directory: source_directory\r\n run: '#filter_tool'\r\n out:\r\n - file_list\r\n ruff_step:\r\n in:\r\n file_list: filter_ruff_step/file_list\r\n source_directory: source_directory\r\n verbose: ruff.verbose\r\n run: '#ruff_tool'\r\n out:\r\n - ruff_report\r\n save_ruff_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: ruff_step/ruff_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n id: ruff_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "ruff": { - "verbose": { - "doc": "Enable verbose logging.", - "type": "boolean", - "label": "Verbose", - "default": false - } - }, - "filter": { - "regex": { - "type": "string", - "label": "regex", - "default": ".*\\.py" - } - } - }, - "version": "0.1", - "tags": [ - 1, - 5 - ], - "tools": [ - "filter", - "ruff", - "save" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "sonarqube", - "fields": { - "name": "SonarQube", - "description": "SonarQube - Code Quality, Security & Static Analysis Tool\r\n\r\nThis tool creates a project in our internal SonarQube server, sends it the code for analysis, and then retrieves the analysis results for storage in the database.", - "pipeline_step": "sonarqube_workflow_step:\r\n in:\r\n pipeline_id: pipeline_id\r\n repo_path: clone_step/repo_directory\r\n run_id: run_id\r\n server_url: server_url\r\n sonarqube_project_key: sonarqube_project_key\r\n sonarqube_project_name: sonarqube_project_name\r\n sonarqube_server: sonarqube_server\r\n sonarqube_token: sonarqube_token\r\n run: '#sonarqube_workflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: sonarqube\r\n pipeline_id:\r\n type: string\r\n repo_path:\r\n type: Directory\r\n run_id:\r\n type: string\r\n server_url:\r\n type: string\r\n sonarqube_project_key:\r\n type: string\r\n sonarqube_project_name:\r\n type: string\r\n sonarqube_server:\r\n type: string\r\n default: sonarqube-sonarqube.sonarqube:9000\r\n sonarqube_token:\r\n type: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n save_sonarqube_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: sonarqube_get_report_step/sonarqube_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n sonarqube_create_project_step:\r\n in:\r\n sonarqube_project_key: sonarqube_project_key\r\n sonarqube_project_name: sonarqube_project_name\r\n sonarqube_server: sonarqube_server\r\n sonarqube_token: sonarqube_token\r\n run: '#sonarqube_create_project_tool'\r\n out:\r\n - sonarqube_project_key\r\n - sonarqube_server\r\n - sonarqube_token\r\n sonarqube_get_report_step:\r\n in:\r\n sonarqube_project_key: sonarqube_scan_step/sonarqube_project_key\r\n sonarqube_server: sonarqube_scan_step/sonarqube_server\r\n sonarqube_token: sonarqube_scan_step/sonarqube_token\r\n run: '#sonarqube_get_report_tool'\r\n out:\r\n - sonarqube_project_key\r\n - sonarqube_server\r\n - sonarqube_token\r\n - sonarqube_report\r\n sonarqube_scan_step:\r\n in:\r\n sonarqube_project_key: sonarqube_create_project_step/sonarqube_project_key\r\n sonarqube_server: sonarqube_create_project_step/sonarqube_server\r\n sonarqube_token: sonarqube_create_project_step/sonarqube_token\r\n source_directory: repo_path\r\n run: '#sonarqube_scan_tool'\r\n out:\r\n - sonarqube_project_key\r\n - sonarqube_server\r\n - sonarqube_token\r\n id: sonarqube_workflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": {}, - "version": "0.1", - "tags": [], - "tools": [ - "sonarqube_create_project", - "sonarqube_scan", - "sonarqube_get_report", - "save" - ] - } - }, - { - "model": "backend.subworkflow", - "pk": "trivy_subworkflow", - "fields": { - "name": "Trivy", - "description": "The all-in-one open source security scanner\r\nUse Trivy to find vulnerabilities (CVE) & misconfigurations (IaC) across code repositories, binary artifacts, container images, Kubernetes clusters, and more.", - "pipeline_step": "trivy_subworkflow:\r\n in:\r\n pipeline_id: pipeline_id\r\n run_id: run_id\r\n server_url: server_url\r\n trivy.image: trivy_subworkflow.trivy.image\r\n run: '#trivy_subworkflow'\r\n out: []", - "definition": "- class: Workflow\r\n\r\n inputs:\r\n name:\r\n type: string\r\n default: trivy\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n trivy.image: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n save_step:\r\n in:\r\n name: name\r\n pipeline_id: pipeline_id\r\n report: trivy_step/trivy_report\r\n run_id: run_id\r\n server_url: server_url\r\n run: '#save_tool'\r\n out: []\r\n trivy_step:\r\n in:\r\n image: trivy.image\r\n run: '#trivy_tool'\r\n out:\r\n - trivy_report\r\n id: trivy_subworkflow\r\n{% for tool in tools %}{{ tool.definition }}\r\n{% endfor %}", - "user_params": { - "trivy": { - "image": { - "type": "string", - "label": "Docker image", - "default": "alpine/git" - } - } - }, - "version": "0.1", - "tags": [ - 6, - 9 - ], - "tools": [ - "save", - "trivy" - ] - } - }, - { - "model": "backend.pipeline", - "pk": 6, - "fields": { - "name": "Python pipeline", - "description": "Runs a series of static analysis tools on python files", - "template": "#!/usr/bin/env cwltool\r\n\r\n$graph:\r\n- class: Workflow\r\n\r\n requirements:\r\n SubworkflowFeatureRequirement: {}\r\n\r\n inputs:\r\n{%- for tool in subworkflows %}\r\n {%- for tool_name, params in tool.user_params.items() %}\r\n {%- for param_name, param in params.items() %}\r\n {{ tool.slug }}.{{ tool_name }}.{{ param_name }}:\r\n label: \"{{ param.label or param_name }}\"\r\n {%- if param.doc %}\r\n doc: |-\r\n {{ param.doc }}\r\n {%- endif %}\r\n type: {{ param.type }}\r\n default: {{ param.default | tojson }}\r\n {%- endfor %}\r\n {%- endfor %}\r\n{%- endfor %}\r\n branch:\r\n type: string\r\n default: ''\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n {% for tool in subworkflows %}{{ tool.pipeline_step | indent(4) }}\r\n {% endfor %}\r\n id: main\r\n{% for tool in subworkflows %}\r\n{{ tool.definition }}\r\n{% endfor %}\r\ncwlVersion: v1.2", - "default_inputs": {}, - "owner": null, - "created_at": null, - "edited_at": null, - "version": "0.1", - "tools": [ - "clone_subworkflow", - "bandit_subworkflow", - "flake8_subworkflow", - "pylint_subworkflow", - "ruff_subworkflow" - ] - } - }, - { - "model": "backend.pipeline", - "pk": 8, - "fields": { - "name": "Notebook pipeline", - "description": "Runs a static analysis and then executes Jupyter Notebooks files (ipynb).", - "template": "#!/usr/bin/env cwltool\r\n\r\n$graph:\r\n- class: Workflow\r\n\r\n requirements:\r\n SubworkflowFeatureRequirement: {}\r\n\r\n inputs:\r\n{%- for tool in subworkflows %}\r\n {%- for tool_name, params in tool.user_params.items() %}\r\n {%- for param_name, param in params.items() %}\r\n {{ tool.slug }}.{{ tool_name }}.{{ param_name }}:\r\n label: \"{{ param.label or param_name }}\"\r\n {%- if param.doc %}\r\n doc: |-\r\n {{ param.doc }}\r\n {%- endif %}\r\n type: {{ param.type }}\r\n default: {{ param.default | tojson }}\r\n {%- endfor %}\r\n {%- endfor %}\r\n{%- endfor %}\r\n branch:\r\n type: string\r\n default: ''\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n {% for tool in subworkflows %}{{ tool.pipeline_step | indent(4) }}\r\n {% endfor %}\r\n id: main\r\n{% for tool in subworkflows %}\r\n{{ tool.definition }}\r\n{% endfor %}\r\ncwlVersion: v1.2", - "default_inputs": {}, - "owner": null, - "created_at": null, - "edited_at": null, - "version": "0.1", - "tools": [ - "clone_subworkflow", - "notebook-bp-validator_subworkflow", - "papermill_subworkflow", - "ruff_ipynb_subworkflow" - ] - } - }, - { - "model": "backend.pipeline", - "pk": 11, - "fields": { - "name": "Docker pipeline", - "description": "Finds vulnerabilities and misconfigurations in Docker images.", - "template": "#!/usr/bin/env cwltool\r\n\r\n$graph:\r\n- class: Workflow\r\n\r\n requirements:\r\n SubworkflowFeatureRequirement: {}\r\n\r\n inputs:\r\n{%- for tool in subworkflows %}\r\n {%- for tool_name, params in tool.user_params.items() %}\r\n {%- for param_name, param in params.items() %}\r\n {{ tool.slug }}.{{ tool_name }}.{{ param_name }}:\r\n label: \"{{ param.label or param_name }}\"\r\n {%- if param.doc %}\r\n doc: |-\r\n {{ param.doc }}\r\n {%- endif %}\r\n type: {{ param.type }}\r\n default: {{ param.default | tojson }}\r\n {%- endfor %}\r\n {%- endfor %}\r\n{%- endfor %}\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n {% for tool in subworkflows %}{{ tool.pipeline_step | indent(4) }}\r\n {% endfor %}\r\n id: main\r\n{% for tool in subworkflows %}\r\n{{ tool.definition }}\r\n{% endfor %}\r\ncwlVersion: v1.2", - "default_inputs": {}, - "owner": null, - "created_at": null, - "edited_at": null, - "version": "0.1", - "tools": [ - "trivy_subworkflow" - ] - } - }, - { - "model": "backend.pipeline", - "pk": 17, - "fields": { - "name": "Notebook static analysis pipeline", - "description": "Runs static analysis on Jupyer Notebook files (ipynb)", - "template": "#!/usr/bin/env cwltool\r\n\r\n$graph:\r\n- class: Workflow\r\n\r\n requirements:\r\n SubworkflowFeatureRequirement: {}\r\n\r\n inputs:\r\n{%- for tool in subworkflows %}\r\n {%- for tool_name, params in tool.user_params.items() %}\r\n {%- for param_name, param in params.items() %}\r\n {{ tool.slug }}.{{ tool_name }}.{{ param_name }}:\r\n label: \"{{ param.label or param_name }}\"\r\n {%- if param.doc %}\r\n doc: |-\r\n {{ param.doc }}\r\n {%- endif %}\r\n type: {{ param.type }}\r\n default: {{ param.default | tojson }}\r\n {%- endfor %}\r\n {%- endfor %}\r\n{%- endfor %}\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n {% for tool in subworkflows %}{{ tool.pipeline_step | indent(4) }}\r\n {% endfor %}\r\n id: main\r\n{% for tool in subworkflows %}\r\n{{ tool.definition }}\r\n{% endfor %}\r\ncwlVersion: v1.2", - "default_inputs": {}, - "owner": null, - "created_at": "2025-09-16T07:47:34.320Z", - "edited_at": "2025-11-18T10:55:05.144Z", - "version": "0.1", - "tools": [ - "clone_subworkflow", - "notebook-bp-validator_subworkflow" - ] - } - }, - { - "model": "backend.pipeline", - "pk": 18, - "fields": { - "name": "Ruff for python", - "description": "Ruff for python", - "template": "#!/usr/bin/env cwltool\n\n$graph:\n- class: Workflow\n\n requirements:\n SubworkflowFeatureRequirement: {}\n\n inputs:\n{%- for tool in subworkflows %}\n {%- for tool_name, params in tool.user_params.items() %}\n {%- for param_name, param in params.items() %}\n {{ tool.slug }}.{{ tool_name }}.{{ param_name }}:\n label: \"{{ param.label or param_name }}\"\n {%- if param.doc %}\n doc: |-\n {{ param.doc }}\n {%- endif %}\n type: {{ param.type }}\n default: {{ param.default | tojson }}\n {%- endfor %}\n {%- endfor %}\n{%- endfor %}\n pipeline_id: string\n run_id: string\n server_url: string\n\n outputs: []\n\n steps:\n {% for tool in subworkflows %}{{ tool.pipeline_step | indent(4) }}\n {% endfor %}\n id: main\n{% for tool in subworkflows %}\n{{ tool.definition }}\n{% endfor %}\ncwlVersion: v1.2\n", - "default_inputs": {}, - "owner": null, - "created_at": "2025-10-21T00:42:55.449Z", - "edited_at": "2025-10-21T00:42:55.449Z", - "version": "0.1", - "tools": [ - "clone_subworkflow", - "ruff_subworkflow" - ] - } - }, - { - "model": "backend.pipeline", - "pk": 19, - "fields": { - "name": "Notebook execution pipeline", - "description": "Executes selected notebooks using Papermill", - "template": "#!/usr/bin/env cwltool\r\n\r\n$graph:\r\n- class: Workflow\r\n\r\n requirements:\r\n SubworkflowFeatureRequirement: {}\r\n\r\n inputs:\r\n{%- for tool in subworkflows %}\r\n {%- for tool_name, params in tool.user_params.items() %}\r\n {%- for param_name, param in params.items() %}\r\n {{ tool.slug }}.{{ tool_name }}.{{ param_name }}:\r\n label: \"{{ param.label or param_name }}\"\r\n {%- if param.doc %}\r\n doc: |-\r\n {{ param.doc }}\r\n {%- endif %}\r\n type: {{ param.type }}\r\n default: {{ param.default | tojson }}\r\n {%- endfor %}\r\n {%- endfor %}\r\n{%- endfor %}\r\n pipeline_id: string\r\n run_id: string\r\n server_url: string\r\n\r\n outputs: []\r\n\r\n steps:\r\n {% for tool in subworkflows %}{{ tool.pipeline_step | indent(4) }}\r\n {% endfor %}\r\n id: main\r\n{% for tool in subworkflows %}\r\n{{ tool.definition }}\r\n{% endfor %}\r\ncwlVersion: v1.2", - "default_inputs": {}, - "owner": null, - "created_at": "2025-11-04T11:59:57.031Z", - "edited_at": "2025-11-25T12:05:05.664Z", - "version": "0.1", - "tools": [ - "papermill_subworkflow", - "clone_subworkflow" - ] - } - } -] \ No newline at end of file diff --git a/backend/backend/migrations/0007_triggertype.py b/backend/backend/migrations/0007_triggertype.py new file mode 100644 index 0000000..61c026e --- /dev/null +++ b/backend/backend/migrations/0007_triggertype.py @@ -0,0 +1,25 @@ +# Generated by Django 5.1.8 on 2026-03-17 16:51 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('backend', '0006_tool_subworkflow_status_available'), + ] + + operations = [ + migrations.CreateModel( + name='TriggerType', + fields=[ + ('slug', models.SlugField(primary_key=True, serialize=False)), + ('description', models.TextField(null=False)), + ('name', models.CharField(max_length=100)), + ('event_type_prefix', models.CharField(max_length=100)), + ('data', models.JSONField(blank=True, default=dict)), + ('status', models.CharField(choices=[('Disabled', 'Disabled'), ('Testing', 'Testing'), ('Restricted', 'Restricted'), ('Enabled', 'Enabled'), ('Deleted', 'Deleted')], default='Enabled', max_length=20)), + ('available', models.BooleanField(default=True)), + ], + ), + ] diff --git a/backend/backend/migrations/0008_trigger_triggerevent.py b/backend/backend/migrations/0008_trigger_triggerevent.py new file mode 100644 index 0000000..84feaed --- /dev/null +++ b/backend/backend/migrations/0008_trigger_triggerevent.py @@ -0,0 +1,44 @@ +# Generated by Django 5.1.8 on 2026-03-18 15:36 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('backend', '0007_triggertype'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='Trigger', + fields=[ + ('slug', models.SlugField(primary_key=True, serialize=False)), + ('description', models.TextField(null=True)), + ('cql2_filter', models.JSONField(blank=True, default=dict, help_text='Filter expressed in CQL2-JSON', verbose_name='CQL2-JSON filter')), + ('params_default', models.JSONField(blank=True, default=dict)), + ('params_mapping', models.JSONField(blank=True, default=dict)), + ('status', models.CharField(choices=[('Disabled', 'Disabled'), ('Testing', 'Testing'), ('Restricted', 'Restricted'), ('Enabled', 'Enabled'), ('Deleted', 'Deleted')], default='Enabled', max_length=20)), + ('enabled', models.BooleanField(default=True)), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.SET_NULL, related_name='triggers', blank=True, null=True, to=settings.AUTH_USER_MODEL)), + ('pipeline', models.ForeignKey(on_delete=django.db.models.deletion.SET_NULL, related_name='triggers', blank=True, null=True, to='backend.pipeline')), + ('trigger_type', models.ForeignKey(on_delete=django.db.models.deletion.SET_NULL, related_name='triggers', blank=True, null=True, to='backend.triggertype')), + ], + ), + migrations.CreateModel( + name='TriggerEvent', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('source', models.CharField(max_length=100)), + ('event_time', models.DateTimeField()), + ('event_type', models.CharField(max_length=100)), + ('event_headers', models.JSONField(blank=True, default=dict)), + ('event_body', models.JSONField(blank=True, default=dict)), + ('pipeline_run', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='triggered_by', to='backend.pipelinerun')), + ('trigger', models.ForeignKey(on_delete=django.db.models.deletion.SET_NULL, related_name='trigger_events', blank=True, null=True, to='backend.trigger')), + ], + ), + ] \ No newline at end of file diff --git a/backend/backend/migrations/0009_pipelinerun_and_jobreport_digest.py b/backend/backend/migrations/0009_pipelinerun_and_jobreport_digest.py new file mode 100644 index 0000000..037b1e1 --- /dev/null +++ b/backend/backend/migrations/0009_pipelinerun_and_jobreport_digest.py @@ -0,0 +1,25 @@ +# Generated by Django 5.1.8 on 2026-07-03 11:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('backend', '0008_trigger_triggerevent'), + ] + + operations = [ + migrations.AddField( + model_name='pipelinerun', + name='digest', + field=models.JSONField(blank=True, default={}), + preserve_default=False, + ), + migrations.AddField( + model_name='jobreport', + name='digest', + field=models.JSONField(default={}), + preserve_default=False, + ), + ] \ No newline at end of file diff --git a/backend/backend/migrations/0010_pipeline_quality_rules.py b/backend/backend/migrations/0010_pipeline_quality_rules.py new file mode 100644 index 0000000..1e224ec --- /dev/null +++ b/backend/backend/migrations/0010_pipeline_quality_rules.py @@ -0,0 +1,18 @@ +# Generated by Django 5.1.8 on 2026-07-05 12:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('backend', '0009_pipelinerun_and_jobreport_digest'), + ] + + operations = [ + migrations.AddField( + model_name='pipeline', + name='quality_rules', + field=models.JSONField(blank=True, default=dict, null=True), + ), + ] diff --git a/backend/backend/models.py b/backend/backend/models.py index af320fd..dd02be8 100644 --- a/backend/backend/models.py +++ b/backend/backend/models.py @@ -12,6 +12,7 @@ class Pipeline(models.Model): created_at = models.DateTimeField(auto_now_add=True, blank=True, null=True) edited_at = models.DateTimeField(auto_now=True, blank=True, null=True) version = models.CharField(max_length=50, null=True) + quality_rules = models.JSONField(default=dict, blank=True, null=True) class Meta: constraints = [ @@ -28,6 +29,7 @@ def __str__(self): class PipelineRun(models.Model): pipeline = models.ForeignKey(Pipeline, related_name="runs", on_delete=models.CASCADE) usage_report = models.JSONField(blank=True) + digest = models.JSONField(blank=True) start_time = models.DateTimeField(blank=True) completion_time = models.DateTimeField(blank=True, null=True) status = models.CharField(max_length=100, blank=True) @@ -41,6 +43,11 @@ class PipelineRun(models.Model): def job_reports_count(self): return self.jobreports.count() + @property + def digest_quality(self): + # Possible values: undefined, unknown, pass, pass_with_comments, fail, exception + return self.digest.get("digest_quality", "undefined") if self.digest else "undefined" + def __str__(self): return f"{'✅' if self.status == 'succeeded' else '❌'} Run {self.id}: {self.pipeline.name}" @@ -49,6 +56,7 @@ class JobReport(models.Model): run = models.ForeignKey(PipelineRun, related_name="jobreports", on_delete=models.CASCADE) name = models.SlugField(max_length=50) output = models.JSONField() + digest = models.JSONField() created_at = models.DateTimeField(null=True) instance = models.CharField(max_length=200, default="") @@ -95,4 +103,63 @@ class CommandLineTool(models.Model): version = models.CharField(max_length=50, null=True) def __str__(self): - return self.name \ No newline at end of file + return self.name + + +class TriggerType(models.Model): + + class Status(models.TextChoices): + DISABLED = 'Disabled' + TESTING = 'Testing' + RESTRICTED = 'Restricted' + ENABLED = 'Enabled' + DELETED = 'Deleted' + + slug = models.SlugField(primary_key=True, max_length=50) + description = models.TextField(null=False) + name = models.CharField(max_length=100, null=False, blank=False) + event_type_prefix = models.CharField(max_length=100, null=False, blank=False) + data = models.JSONField(default=dict, blank=True) + status = models.CharField(max_length=20, choices=Status.choices, default=Status.ENABLED) + available = models.BooleanField(default=True) + + def __str__(self): + return self.name + + +class Trigger(models.Model): + + class Status(models.TextChoices): + DISABLED = 'Disabled' + TESTING = 'Testing' + RESTRICTED = 'Restricted' + ENABLED = 'Enabled' + DELETED = 'Deleted' + + slug = models.SlugField(primary_key=True, max_length=50) + description = models.TextField(null=True) + cql2_filter = models.JSONField(default=dict, blank=True, help_text="Filter expressed in CQL2-JSON", verbose_name="CQL2-JSON filter") + params_default = models.JSONField(default=dict, blank=True) + params_mapping = models.JSONField(default=dict, blank=True) + trigger_type = models.ForeignKey(TriggerType, related_name="triggers", blank=True, null=True, on_delete=models.SET_NULL) + pipeline = models.ForeignKey(Pipeline, related_name="triggers", blank=True, null=True, on_delete=models.SET_NULL) + owner = models.ForeignKey(User, related_name="triggers", blank=True, null=True, on_delete=models.SET_NULL) + status = models.CharField(max_length=20, choices=Status.choices, default=Status.ENABLED) + enabled = models.BooleanField(default=True) + + def __str__(self): + return f"Trigger of type: {self.trigger_type.name}, Pipeline: {self.pipeline.name}, Pipeline version: {self.pipeline.version}" + + +class TriggerEvent(models.Model): + trigger = models.ForeignKey(Trigger, related_name="trigger_events", blank=True, null=True, on_delete=models.SET_NULL) + source = models.CharField(max_length=100, null=False, blank=False) + event_time = models.DateTimeField(auto_now_add=False, blank=False, null=False) + event_type = models.CharField(max_length=100, null=False, blank=False) + event_headers = models.JSONField(default=dict, blank=True) # Stored, e.g. a CloudEvent headers + event_body = models.JSONField(default=dict, blank=True) # Stored, e.g. a CloudEvent body + # Note: This should have been implemented using a OneToOneField as a un can have at most one associated event + pipeline_run = models.ForeignKey(PipelineRun, on_delete=models.SET_NULL, blank=True, null=True, related_name="triggered_by") + + def __str__(self): + return f"Trigger Event {self.id}: {self.trigger}, Run: {self.pipeline_run}" diff --git a/backend/backend/serializers.py b/backend/backend/serializers.py index 40406ce..e2e2823 100644 --- a/backend/backend/serializers.py +++ b/backend/backend/serializers.py @@ -1,5 +1,28 @@ from rest_framework import serializers -from backend.models import Pipeline, PipelineRun, JobReport, Subworkflow, Tag +from django.contrib.auth.models import User + +from backend.models import ( + Pipeline, + PipelineRun, + JobReport, + Subworkflow, + Tag, + TriggerType, + Trigger, + TriggerEvent, +) + + +class UserMinifiedSerializer(serializers.ModelSerializer): + full_name = serializers.SerializerMethodField() + + class Meta: + model = User + fields = ("id", "username", "first_name", "last_name", "full_name") + + def get_full_name(self, obj): + name = f"{obj.first_name} {obj.last_name}".strip() + return name if name else obj.username class PipelineSerializer(serializers.ModelSerializer): @@ -13,6 +36,7 @@ class Meta: "description", "tools", "default_inputs", + "quality_rules", "owner", "owner_name", "created_at", @@ -22,8 +46,10 @@ class Meta: class PipelineRunSerializer(serializers.ModelSerializer): - job_reports_count = serializers.SerializerMethodField() + digest_quality = serializers.SerializerMethodField() started_by = serializers.ReadOnlyField(source="started_by.username") + trigger_event = serializers.SerializerMethodField() + job_reports_count = serializers.SerializerMethodField() class Meta: model = PipelineRun @@ -37,19 +63,30 @@ class Meta: "user", "inputs", "output", + "digest", + "digest_quality", "executed_cwl", "started_by", + "trigger_event", "job_reports_count", ] def get_job_reports_count(self, obj): return obj.job_reports_count + def get_digest_quality(self, obj): + return obj.digest_quality + + def get_trigger_event(self, obj): + # Fetch the first trigger event from the relation if it exists + first_event = obj.triggered_by.first() + return first_event.id if first_event else None + class JobReportSerializer(serializers.ModelSerializer): class Meta: model = JobReport - fields = ["id", "name", "instance", "created_at", "output", "run"] + fields = ["id", "name", "instance", "created_at", "output", "digest", "run"] read_only_fields = ["run"] @@ -73,3 +110,90 @@ class TagSerializer(serializers.ModelSerializer): class Meta: model = Tag fields = ["id", "name"] + + +class TriggerTypeSerializer(serializers.ModelSerializer): + class Meta: + model = TriggerType + fields = ["slug", "name", "description", "status", "data"] + + +class TriggerSerializer(serializers.ModelSerializer): + trigger_type = serializers.SlugRelatedField( + slug_field="slug", + queryset=TriggerType.objects.all() + ) + trigger_type_name = serializers.ReadOnlyField(source="trigger_type.name") + pipeline_id = serializers.PrimaryKeyRelatedField( + source="pipeline", + queryset=Pipeline.objects.all() + ) + pipeline_name = serializers.ReadOnlyField(source="pipeline.name") + pipeline_version = serializers.ReadOnlyField(source="pipeline.version") + owner = UserMinifiedSerializer(read_only=True) + owner_name = serializers.CharField(write_only=True, required=False, allow_null=True) + event_count = serializers.IntegerField(read_only=True) + + class Meta: + model = Trigger + fields = [ + "slug", + "owner", + "owner_name", + "description", + "enabled", + "status", + "cql2_filter", + "params_default", + "params_mapping", + "trigger_type", + "trigger_type_name", + "pipeline_id", + "pipeline_name", + "pipeline_version", + "event_count", + ] + + def to_internal_value(self, data): + internal_value = super().to_internal_value(data) + # If present, replace the "owner_name" with the corresponding User instance + owner_name = internal_value.pop("owner_name", None) + if owner_name: + try: + user_instance = User.objects.get(username=owner_name) + internal_value["owner"] = user_instance + except User.DoesNotExist: + raise serializers.ValidationError({ + "owner": [f"User '{owner_name}' does not exist."], + }) + return internal_value + + +class TriggerEventSerializer(serializers.ModelSerializer): + trigger = serializers.ReadOnlyField(source="trigger.slug") + trigger_type = serializers.ReadOnlyField(source="trigger.trigger_type.slug") + trigger_type_name = serializers.ReadOnlyField(source="trigger.trigger_type.name") + pipeline_run_id = serializers.ReadOnlyField(source="pipeline_run.id") + pipeline_id = serializers.ReadOnlyField(source="pipeline_run.pipeline.id") + pipeline_name = serializers.ReadOnlyField(source="pipeline_run.pipeline.name") + pipeline_version = serializers.ReadOnlyField(source="pipeline_run.pipeline.version") + user = serializers.ReadOnlyField(source="pipeline_run.user.username") + + class Meta: + model = TriggerEvent + fields = [ + "id", + "source", + "event_time", + "event_type", + "event_headers", + "event_body", + "user", + "trigger", + "trigger_type", + "trigger_type_name", + "pipeline_run_id", + "pipeline_id", + "pipeline_name", + "pipeline_version", + ] diff --git a/backend/backend/urls.py b/backend/backend/urls.py index 55386d3..dc22454 100644 --- a/backend/backend/urls.py +++ b/backend/backend/urls.py @@ -8,15 +8,23 @@ TagViewSet, SettingsView, EventsView, + TriggerTypeViewSet, + TriggerViewSet, + TriggerEventViewSet, + TriggerRunViewSet, ) router = DefaultRouter() -router.register(r"pipelines", PipelineViewSet, basename="pipeline") -router.register(r"pipelines/(?P[^/.]+)/runs", PipelineRunViewSet, basename="pipeline-run") -router.register(r"pipelines/(?P[^/.]+)/runs/(?P[^/.]+)/jobreports", JobReportViewSet, basename="pipeline-run-jobreport") -router.register(r"tools", SubworkflowViewSet, basename="tool") -router.register(r"tags", TagViewSet, basename="tag") +router.register(r"pipelines", PipelineViewSet, basename="pipeline") +router.register(r"pipelines/(?P[^/.]+)/runs", PipelineRunViewSet, basename="pipeline-run") +router.register(r"pipelines/(?P[^/.]+)/runs/(?P[^/.]+)/jobreports", JobReportViewSet, basename="pipeline-run-jobreport") +router.register(r"tools", SubworkflowViewSet, basename="tool") +router.register(r"tags", TagViewSet, basename="tag") +router.register(r"triggertypes", TriggerTypeViewSet, basename="trigger-type") +router.register(r"triggers", TriggerViewSet, basename="trigger") +router.register(r"triggers/(?P[^/.]+)/events", TriggerEventViewSet, basename="trigger-event") +router.register(r"triggers/(?P[^/.]+)/runs", TriggerRunViewSet, basename="trigger-run") urlpatterns = router.urls diff --git a/backend/backend/utils/cloudevents.py b/backend/backend/utils/cloudevents.py index 24e1089..3254bfd 100644 --- a/backend/backend/utils/cloudevents.py +++ b/backend/backend/utils/cloudevents.py @@ -1,7 +1,5 @@ import json import logging -import os -import time from cloudevents.conversion import to_structured from cloudevents.http import CloudEvent @@ -10,7 +8,7 @@ logger = logging.getLogger(__name__) -def decode(body, headers: dict) -> tuple[dict, dict]: +def decode(body: str, headers: dict) -> tuple[dict, dict]: """ Decode CloudEvent and return the payload and the headers """ @@ -36,17 +34,14 @@ def decode(body, headers: dict) -> tuple[dict, dict]: logger.debug("Event Headers:\n%s", json.dumps(headers_dict, indent=2)) logger.debug("Event Data:\n%s", json.dumps(payload_dict, indent=2)) - except json.JSONDecodeError: - logger.error("JSONDecodeError") - return Response( - {"error": "Invalid JSON payload in event body"}, - status=status.HTTP_400_BAD_REQUEST - ) + except json.JSONDecodeError as error: + logger.error("JSONDecodeError: %s", error) + raise error return payload_dict, headers_dict -def encode(attributes, data, headers=None): +def encode(attributes: dict, data: dict): event = CloudEvent(attributes, data) logger.debug("Event: %s", event) _ignore, payload = to_structured(event) @@ -56,4 +51,50 @@ def encode(attributes, data, headers=None): } logger.debug("Headers: %s", headers_dict) logger.debug("Payload: %s", payload) - return payload, headers_dict \ No newline at end of file + return payload, headers_dict + + +def is_match(filter_node: dict, payload: dict) -> bool: + """ + Evaluates a CQL2-JSON filter against a Python dictionary. + Supports 'and', 'or', and '=' operators with nested property access. + Example filter: + ```json + { + "op": "and", + "args": [ + {"op": "=", "args": [{"property": "repository.full_name"}, "SpaceApplications/eoepca-aqbb-test-files"]}, + {"op": "=", "args": [{"property": "ref"}, "refs/heads/main"]} + ] + } + ``` + """ + logger.debug("Filter node: %s", filter_node) + op = filter_node.get("op", "").lower() + args = filter_node.get("args", []) + # Handle Logical Operators + if op == "and": + return all(is_match(arg, payload) for arg in args) + if op == "or": + return any(is_match(arg, payload) for arg in args) + # Handle Comparison Operators + if op == "=": + left_raw = args[0] + right = args[1] # The constant value + # Resolve the property value (e.g., "repository.full_name") + left_val = None + if isinstance(left_raw, dict) and "property" in left_raw: + path = left_raw["property"].split('.') + left_val = payload + for part in path: + if isinstance(left_val, dict): + left_val = left_val.get(part) + else: + left_val = None + break + else: + left_val = left_raw + + return left_val == right + # Default to False for unknown operators + return False diff --git a/backend/backend/utils/github.py b/backend/backend/utils/github.py new file mode 100644 index 0000000..0f81af5 --- /dev/null +++ b/backend/backend/utils/github.py @@ -0,0 +1,104 @@ +import logging +import os +import requests + +from enum import Enum +from .tools import getenv_bool + + +logger = logging.getLogger(__name__) + + +GITHUB_STATUS_ENABLED = getenv_bool("GITHUB_STATUS_ENABLED", False) +if GITHUB_STATUS_ENABLED: + GITHUB_STATUS_CONTEXT = os.getenv("GITHUB_STATUS_CONTEXT", "EOEPCA Application Quality / Quality Check") + GITHUB_STATUS_DESCRIPTION = os.getenv("GITHUB_STATUS_DESCRIPTION", "Application quality metrics met all threshold guidelines.") + GITHUB_STATUS_TARGET_URL = os.getenv("GITHUB_STATUS_TARGET_URL", os.getenv("PUBLIC_URL")) + + +class GH_CONTEXT_STATUS(str, Enum): + # Statuses supported by GitHub + # https://docs.github.com/en/rest/commits/statuses + PENDING = "pending" + SUCCESS = "success" + FAILURE = "failure" + ERROR = "error" + + def __str__(self) -> str: + return self.value + + @classmethod + def has_value(cls, value: str) -> bool: + return value in cls._value2member_map_ + + @classmethod + def describe(cls, value: str) -> bool: + if value == GH_CONTEXT_STATUS.PENDING: + return "Best practices validation is being applied" + elif value == GH_CONTEXT_STATUS.FAILURE: + return "Best practices validation failed" + elif value == GH_CONTEXT_STATUS.SUCCESS: + return "Best practices validation is successful" + # Remaining value is ERROR + return "Failed to apply best practices validation (internal error)" + + +def post_quality_state(owner, repo, sha, state, statuses_url=None, target_url=None, description=None): + if not GITHUB_STATUS_ENABLED: + msg = "GitHub Status is disabled. Not posting status %s to %s/%s" + logger.warning(msg, state, owner, repo) + return None + # Try to get the GitHub owner(organisation)-specific API token, if defined + GITHUB_API_TOKEN = os.getenv(f"GITHUB_API_TOKEN__{owner}", None) + # Otherwise, try to get a global API token + if not GITHUB_API_TOKEN: + logger.warning("No specific GitHub API token found for owner %s", owner) + GITHUB_API_TOKEN = os.getenv(f"GITHUB_API_TOKEN", None) + if not GITHUB_API_TOKEN: + logger.error("Missing GitHub API token for posting status to %s/%s", owner, repo) + return None + headers = { + "Authorization": "Bearer " + GITHUB_API_TOKEN, + "Accept": "application/vnd.github+json", + } + if not GH_CONTEXT_STATUS.has_value(state): + logger.error("Invalid GitHub status: %s", state) + # If a GitHub API URL is not provided, generate one + if not statuses_url: + statuses_url = f"https://api.github.com/repos/{owner}/{repo}/statuses/{sha}" + if not description: + description = GH_CONTEXT_STATUS.describe(state) + payload = { + "state": state, + "context": GITHUB_STATUS_CONTEXT, + "description": description or GITHUB_STATUS_DESCRIPTION, + "target_url": target_url or GITHUB_STATUS_TARGET_URL, + } + + response = requests.post(statuses_url, json=payload, headers=headers) + if response.status_code >= 200 and response.status_code < 300: + msg = "GitHub status updated for %s/%s: %s = %s" + logger.info(msg, owner, repo, GITHUB_STATUS_CONTEXT, state) + elif response.status_code >= 400: + msg = "Failed to updated GitHub status for %s/%s: %s = %s \n%s" + err = json.dumps(response.json, indent=2) + logger.error(msg, owner, repo, GITHUB_STATUS_CONTEXT, state, err) + return response + + +def get_properties(event_body): + # Extract the owner, repository, and commit SHA from the event body + owner = None + repo = None + sha = None + if "pusher" in event_body: + owner = event_body.get("repository", {}).get("owner", None).get("login", None) + repo = event_body.get("repository", {}).get("name", None) + sha = event_body.get("head_commit", {}).get("id", None) + elif "pull_request" in event_body: + # if event_body["action"] => "opened" | "synchronize" + owner = event_body.get("repository", {}).get("owner", None).get("login", None) + repo = event_body.get("repository", {}).get("name", None) + sha = event_body.get("pull_request", {}).get("head", {}).get("sha", {}) + logger.debug("GitHub event properties: owner=%s, repo=%s, sha=%s", owner, repo, sha) + return owner, repo, sha \ No newline at end of file diff --git a/backend/backend/utils/run_workflow.py b/backend/backend/utils/run_workflow.py index fbd6b7a..a7e5c5a 100644 --- a/backend/backend/utils/run_workflow.py +++ b/backend/backend/utils/run_workflow.py @@ -1,6 +1,7 @@ import logging import os +from datetime import datetime from json.decoder import JSONDecodeError from kubernetes import config @@ -9,6 +10,12 @@ from pycalrissian.context import CalrissianContext from pycalrissian.execution import CalrissianExecution from pycalrissian.job import CalrissianJob +from rule_engine import Rule +from .github import ( + GH_CONTEXT_STATUS, + post_quality_state as gh_post_quality_state, + get_properties as gh_get_properties, +) from .tools import getenv_bool @@ -45,22 +52,39 @@ logger = logging.getLogger(__name__) -def run_workflow( - parameters: dict, - run_id: int, - cwl: dict, - username: str, -) -> dict: - pipeline_run = PipelineRun.objects.get(id=run_id) +def _get_cluster_config_file(callback_url: str, username: str = "") -> str: + cluster_config_file = None - ''' - """ - Create the image pull secrets - """ - username = "" - password = "" + if WORKSPACE_VCLUSTER_ENABLED: + try: + cluster_config_file = get_vcluster_config_file("ws-" + username) + # Use the public URL if the pipeline is run in a vCluster + callback_url = PUBLIC_URL + # Saving the vCluster kubeconfig in a file allows debugging with e.g. k9s + logger.debug("Workspace vCluster kubeconfig file: %s", cluster_config_file) + except Exception as e: + logger.error("Failed to obtain the Workspace vCluster config: %s", e) + cluster_config_file = None + if WORKSPACE_VCLUSTER_REQUIRED: + logger.error("Workspace vCluster is required. Aborting the execution") + raise + if cluster_config_file is None and SHARED_VCLUSTER_ENABLED: + try: + cluster_config_file = get_vcluster_config_file("application-quality-vcluster") + callback_url = PUBLIC_URL + logger.debug("Shared vCluster kubeconfig file: %s", cluster_config_file) + except Exception as e: + logger.error("Failed to obtain the Shared vCluster config: %s", e) + cluster_config_file = None + if SHARED_VCLUSTER_REQUIRED: + logger.error("Shared vCluster is required. Aborting the execution") + raise + return callback_url, cluster_config_file + + +def _create_image_pull_secrets(registry: str, username: str, password: str) -> dict: """ Make a string with the username and the password, turn it into a string literal (encode()), @@ -68,14 +92,188 @@ def run_workflow( turn the encoded string literal back into a regular string (decode()). """ auth = base64.b64encode(f"{username}:{password}".encode()).decode() - - secret_config = { + return { "auths": { "registry.gitlab.com": {"auth": ""}, "https://index.docker.io/v1/": {"auth": ""}, + registry: {"auth": auth}, } } - ''' + + +def _check_digest_rules(pipeline_run: PipelineRun, pr_digest: dict) -> str: + """ + Example rule: + "(error == 0 and critical == 0) or security < 1" + """ + logger.info("Check quality rules to pipeline run %s", pipeline_run.id) + quality_rules = pr_digest.get("quality_rules", {}) + logger.info("Quality rules: %s", quality_rules) + digest_issues = pr_digest.get("issues", {}) + result = None + try: + # Check the "pass_with_comments" rule, if present + if "pass_with_comments" in quality_rules: + rule = Rule(quality_rules["pass_with_comments"]) + logger.debug("Checking pass_with_comments rule: %s", rule) + if rule.matches(digest_issues): + result = "pass_with_comments" + if result is None: + if "pass" in quality_rules: + rule = Rule(quality_rules["pass"]) + logger.debug("Checking pass rule: %s", rule) + if rule.matches(digest_issues): + result = "pass" + else: + result = "fail" + else: + # No "pass" rule => Cannot determine the quality + logger.debug("'pass' rule required to deduce the quality.") + result = "unknown" + except Exception as e: + logger.error("Failed to check quality rules: %s", e) + result = "exception" + return result + + +def _generate_pipeline_run_digest(pipeline_run: PipelineRun) -> dict: + logger.debug("Generate digest for pipeline run %s", pipeline_run.id) + pr_digest = { + "properties": { + "name": "", + "version": "", + }, + "issues": { + "info": 0, + "convention": 0, + "warning": 0, + "error": 0, + "security": 0, + "critical": 0, + }, + "quality_rules": None, + "digest_quality": None, + "created_at": datetime.now().isoformat(), + } + # Collect the issue counts from the job reports digest + for job_report in pipeline_run.jobreports.all(): + if job_report.digest: + issues = job_report.digest.get("issues", {}) + for key in ["info", "convention", "warning", "error", "security", "critical"]: + pr_digest["issues"][key] += issues.get(key, 0) + # Retrieve the application quality rules from the pipeline definition + # "pass": "applications with good quality pass that rule", + # "pass_with_comments": "applications with minor warnings/comments pass that rule", + # Otherwise, the quality is "fail" + try: + quality_rules = pipeline_run.pipeline.quality_rules + except Exception as e: + logger.error("Error while retrieving pipeline quality rules: %s", e) + quality_rules = { + "pass": "error == 0 and critical == 0 and security == 0 and warning == 0" + } + pr_digest["quality_rules"] = quality_rules + pr_digest["digest_quality"] = _check_digest_rules(pipeline_run, pr_digest) + return pr_digest + + +def _update_pipeline_run(pipeline_run: PipelineRun, execution: CalrissianExecution): + logger.debug("Update pipeline run %s", pipeline_run.id) + try: + usage = execution.get_usage_report() + except UnboundLocalError: + usage = "Couldn't copy usage report locally" + logger.error(usage) + + try: + output = execution.get_output() + except JSONDecodeError: + output = "Output file contains no JSON" + logger.error(output) + except UnboundLocalError: + output = "Couldn't copy output locally" + logger.error(output) + + logger.info("start time: %s", execution.get_start_time()) + logger.info("completion time: %s", execution.get_completion_time()) + logger.info("complete %s", execution.is_complete()) + logger.info("succeeded %s", execution.is_succeeded()) + # tool_logs = execution.get_tool_logs() # Can be useful to avoid using save_tool + + digest = _generate_pipeline_run_digest(pipeline_run) + pipeline_run.refresh_from_db() + pipeline_run.usage_report = usage + # pipeline_run.start_time = execution.get_start_time() + pipeline_run.completion_time = execution.get_completion_time() + pipeline_run.status = execution.get_status().value + pipeline_run.output = output + pipeline_run.digest = digest + + pipeline_run.save() + logger.info("Pipeline run %s updated", pipeline_run.id) + + if digest.get("digest_quality", None) in ["pass", "pass_with_comments"]: + update_quality_status(pipeline_run, GH_CONTEXT_STATUS.SUCCESS) + else: + update_quality_status(pipeline_run, GH_CONTEXT_STATUS.FAILURE) + + +def update_quality_status(pipeline_run: PipelineRun, status: str): + """ + This function updates in GitHub or GitLab the quality status of the code being analysed. + The pipeline run must be linked to a push event issued by GitHub or GitLab. + The parameters (owner, repository, commit SHA) are extracted from the event body. + Before starting the workflow, the status is set to PENDING. + When the execution is complete, the status depends on the computed digest: SUCCESS or FAILURE. + """ + logger.debug("Update application quality status for Run %s", pipeline_run.id) + response = None + try: + # Obtain the trigger event, if any + event = pipeline_run.triggered_by.first() + if not event: + # This pipeline run has not been triggered by an event + logger.info( + "Run %s not triggered by an event. Skipping quality status update.", + pipeline_run.id, + ) + return + # Extract the owner, repository, and commit SHA from the event body + # and update the quality status in GitHub + if event.event_type.startswith("org.eoepca.webhook.github"): + owner, repo, sha = gh_get_properties(event.event_body) + response = gh_post_quality_state(owner, repo, sha, status) + logger.debug( + "Application quality status updated in GitHub for Run %s", + pipeline_run.id, + ) + elif event.event_type.startswith("org.eoepca.webhook.gitlab"): + # Updating quality status in GitLab is not supported yet + logger.warning("Application quality status in GitLab is not supported yet.") + else: + logger.info( + "Run %s not triggered by a push or pull_request event: %s. Skipping quality status update.", + pipeline_run.id, + event.event_type, + ) + except Exception as e: + logger.error( + "Failed to update application quality status for Run %s: %s", + pipeline_run.id, + e, + ) + raise e + return response + + +def run_workflow( + parameters: dict, + run_id: int, + cwl: dict, + username: str, +) -> dict: + pipeline_run = PipelineRun.objects.get(id=run_id) + update_quality_status(pipeline_run, GH_CONTEXT_STATUS.PENDING) logger.debug("Executing workflow for user %s", username) @@ -96,36 +294,11 @@ def run_workflow( logger.error("Failed to load in-cluster config: %s", e) raise - # Without a config file, PyCalrissian uses the local cluster - cluster_config_file = None # Use the internal backend service URL if the pipeline is run in the local cluster callback_url = f"http://{BACKEND_SERVICE_HOST}:{BACKEND_SERVICE_PORT}" - if WORKSPACE_VCLUSTER_ENABLED: - try: - cluster_config_file = get_vcluster_config_file("ws-" + username) - # Use the public URL if the pipeline is run in a vCluster - callback_url = PUBLIC_URL - # Saving the vCluster kubeconfig in a file allows debugging with e.g. k9s - logger.debug("Workspace vCluster kubeconfig file: %s", cluster_config_file) - except Exception as e: - logger.error("Failed to obtain the Workspace vCluster config: %s", e) - cluster_config_file = None - if WORKSPACE_VCLUSTER_REQUIRED: - logger.error("Workspace vCluster is required. Aborting the execution") - raise - - if cluster_config_file is None and SHARED_VCLUSTER_ENABLED: - try: - cluster_config_file = get_vcluster_config_file("application-quality-vcluster") - callback_url = PUBLIC_URL - logger.debug("Shared vCluster kubeconfig file: %s", cluster_config_file) - except Exception as e: - logger.error("Failed to obtain the Shared vCluster config: %s", e) - cluster_config_file = None - if SHARED_VCLUSTER_REQUIRED: - logger.error("Shared vCluster is required. Aborting the execution") - raise + # Without a config file, PyCalrissian uses the local cluster + callback_url, cluster_config_file = _get_cluster_config_file(callback_url) # If cluster_config_file is None here, the ultimate option (if vclusters are not required) # is running the pipeline in the host cluster @@ -163,7 +336,6 @@ def run_workflow( session.initialise() - # # Create the Calrissian job # https://terradue.github.io/pycalrissian/gettingstarted/#create-the-calrissianjob os.environ["CALRISSIAN_IMAGE"] = AQBB_CALRISSIANIMAGE @@ -196,47 +368,18 @@ def run_workflow( pipeline_run.save(update_fields=["status"]) logger.debug("Run %s status updated: running", pipeline_run.id) - # # Monitoring - # execution.monitor(interval=20) - try: - usage = execution.get_usage_report() - except UnboundLocalError: - usage = "Couldn't copy usage report locally" - logger.error(usage) - try: - output = execution.get_output() - except JSONDecodeError: - output = "Output file contains no JSON" - logger.error(output) - except UnboundLocalError: - output = "Couldn't copy output locally" - logger.error(output) - - logger.info("start time: %s", execution.get_start_time()) - logger.info("completion time: %s", execution.get_completion_time()) - logger.info("complete %s", execution.is_complete()) - logger.info("succeeded %s", execution.is_succeeded()) - # tool_logs = execution.get_tool_logs() # Can be useful to avoid using save_tool + # Update the pipeline run with outputs, resource consumption, digests, quality, etc. + _update_pipeline_run(pipeline_run, execution) - # # Delete the Kubernetes namespace - # if execution.is_succeeded(): session.dispose() else: log = execution.get_log() logger.error("Execution failed for run %s", pipeline_run.id) logger.info(log) - - pipeline_run.refresh_from_db() - pipeline_run.usage_report = usage - # pipeline_run.start_time = execution.get_start_time() - pipeline_run.completion_time = execution.get_completion_time() - pipeline_run.status = execution.get_status().value - pipeline_run.output = output - - pipeline_run.save() + logger.info("Run %s completed", pipeline_run.id) diff --git a/backend/backend/utils/workspaces.py b/backend/backend/utils/workspaces.py index e7673bc..5eb0f2a 100644 --- a/backend/backend/utils/workspaces.py +++ b/backend/backend/utils/workspaces.py @@ -2,7 +2,6 @@ import os import requests import subprocess -import sys WORKSPACE_API_SERVICE_URL = os.getenv( @@ -13,7 +12,7 @@ logger = logging.getLogger(__name__) -def get_vcluster_config(ws_name: str) -> tuple([dict, str]): +def get_vcluster_config(ws_name: str) -> (dict, str): logger.info("Fetching kubeconfig for workspace %s", ws_name) logger.debug("HTTP request URL: %s/workspaces/%s", WORKSPACE_API_SERVICE_URL, ws_name) ws_info = requests.get(f"{WORKSPACE_API_SERVICE_URL}/workspaces/{ws_name}", timeout=10) @@ -27,7 +26,7 @@ def get_vcluster_config(ws_name: str) -> tuple([dict, str]): if ws_info_dict.get("status", "unknown") in [None, "unknown"]: logger.error("Error: workspace %s not found", ws_name) return None, "error" - if not ws_info_dict.get("status", None) == "ready": + if ws_info_dict.get("status", None) != "ready": logger.error("Error: workspace %s is not ready", ws_name) return None, "error" logger.info("Workspace %s is ready", ws_name) @@ -39,7 +38,7 @@ def get_vcluster_config(ws_name: str) -> tuple([dict, str]): ws_cluster_status = ws_cluster_dict.get("status", None) ws_cluster_config = ws_cluster_dict.get("config", None) logger.debug("Workspace cluster status: %s", ws_cluster_status) - if not ws_cluster_status == "active": + if ws_cluster_status != "active": # Other possible statuses: suspended, disabled logger.info("Cluster not active => Must resume or active it now (TODO)") return "", ws_cluster_status @@ -75,4 +74,4 @@ def get_vcluster_config_file(ws_name: str, vc_name: str = "default-vc") -> str: if os.path.getsize(config_file) == 0: raise ValueError("Error: could not obtain vcluster: %s", result.stderr) # Return the name of the file that contains the kubeconfig - return config_file \ No newline at end of file + return config_file diff --git a/backend/backend/views.py b/backend/backend/views.py index 5c23725..b7518d2 100644 --- a/backend/backend/views.py +++ b/backend/backend/views.py @@ -1,21 +1,34 @@ +import json import logging import os import time import yaml from django.utils import timezone +from django.db.models import Count, Q from django.db.utils import IntegrityError from django.contrib.auth.models import User +from django.core.exceptions import ValidationError from jinja2 import Template from rest_framework import mixins, permissions, status, viewsets -from rest_framework.exceptions import ValidationError +from rest_framework.exceptions import PermissionDenied from rest_framework.response import Response from rest_framework.views import APIView -from backend.models import Pipeline, PipelineRun, JobReport, Subworkflow, Tag +from backend.models import ( + Pipeline, + PipelineRun, + JobReport, + Subworkflow, + Tag, + TriggerType, + Trigger, + TriggerEvent +) from backend.tasks import run_workflow_task -from backend.utils.cloudevents import encode, decode +from backend.utils.cloudevents import encode as ce_encode, decode as ce_decode, is_match +from backend.utils.run_workflow import update_quality_status from . import serializers @@ -30,8 +43,8 @@ try: logger.info("Loading %s", pipeline_cwl_template_path) - with open(pipeline_cwl_template_path, "r", encoding="utf-8") as file: - pipeline_cwl_template = file.read() + with open(pipeline_cwl_template_path, "r", encoding="utf-8") as tpl: + pipeline_cwl_template = tpl.read() except Exception as ex: logger.error("Error loading %s: %s", pipeline_cwl_template_path, str(ex)) pipeline_cwl_template = f"Failed to load pipeline CWL template: {ex}" @@ -42,8 +55,8 @@ def get(self, request): settings_file = os.path.abspath("settings.yaml") try: logger.info("Loading %s", settings_file) - with open(settings_file, "r", encoding="utf-8") as f: - settings = yaml.safe_load(f) + with open(settings_file, "r", encoding="utf-8") as file: + settings = yaml.safe_load(file) except FileNotFoundError: logger.info("Not found: %s", settings_file) settings = None @@ -58,29 +71,82 @@ def get(self, request): class EventsView(APIView): + + @staticmethod + def _get_matching_user(event_user, event_sender=None): + logger.debug("Possible users: %s, %s", event_user, event_sender) + user = None + if event_user: + user = User.objects.filter(username=event_user).first() + logger.debug("User (%s): %s", event_user, user) + if not user and event_sender: + user = User.objects.filter(username=event_sender).first() + logger.debug("User (%s): %s", event_sender, user) + if not user: + logger.debug("No user identified in the event") + return user + + @staticmethod + def _get_pipeline_id(event_subject): + pipeline_id = None + if event_subject and event_subject.startswith("pipelines/"): + pipeline_id = event_subject.split("/")[-1] + logger.debug("Pipeline: %s", pipeline_id) + else: + logger.debug("No pipeline identified in the event") + return pipeline_id + + @staticmethod + def _get_matching_triggers(event_type, event_payload): + matching_triggers = [] + # Check if the event_type matches one (or more?) active TriggerType + _trigger_types = TriggerType.objects.filter( + available=True, + status__in=["Testing", "Restricted", "Enabled"] + ) + trigger_types = [ + t for t in _trigger_types if event_type.startswith(t.event_type_prefix) + ] + logger.debug("The event matches these trigger types: %s", trigger_types) + # Retrieve active Triggers, if any + for trigger_type in trigger_types: + triggers = trigger_type.triggers.filter(enabled=True) + logger.debug("Enabled triggers of type '%s': %s", trigger_type, triggers) + for trigger in triggers: + user = trigger.owner + logger.debug("Trigger owner: %s", user) + # Verify if the event matches the filter configured in the trigger + filter_json = trigger.cql2_filter + logger.debug("CQL2-JSON Filter: %s", filter_json) + ce_is_match = is_match(filter_json, event_payload) + logger.debug("CloudEvent is %smatching!", "" if ce_is_match else "not ") + + if ce_is_match: + matching_triggers.append(trigger) + return matching_triggers + def post(self, request, *args, **kwargs): try: logger.info("Event received %s", request) - payload, headers = decode(request.body, request.META.items()) + payload, headers = ce_decode(request.body, request.META.items()) + except json.JSONDecodeError as error: + logger.error("JSONDecodeError: %s", error) + return Response( + {"error": "Invalid JSON payload in event body"}, + status=status.HTTP_400_BAD_REQUEST + ) + try: event_id = headers.get('Ce-Id') + event_source = headers.get('Ce-Source', None) + # event_webhook_source = headers.get('Ce-Webhooksource', None) event_user = headers.get('Ce-User', None) + # In GitHub events, if "sender.type" is "User", then "sender.login" is the user ID + event_sender = payload.get("sender", {}).get("login", None) event_type = headers.get('Ce-Type', None) event_subject = headers.get('Ce-Subject', None) + event_time = headers.get('Ce-Time', None) - user = None - pipeline_id = None - - if event_user: - user = User.objects.get(username=event_user) - logger.debug("User: %s", user) - else: - logger.debug("No user identified in the event") - - if event_subject and event_subject.startswith("pipelines/"): - pipeline_id = event_subject.split("/")[-1] - logger.debug("Pipeline: %s", pipeline_id) - else: - logger.debug("No pipeline identified in the event") + user = self._get_matching_user(event_user, event_sender) # Default response data res_data = { @@ -97,7 +163,55 @@ def post(self, request, *args, **kwargs): } # React to the event - # Note: The Trigger must filter on the event type prefix + # Note: The Knative Trigger filters on the event type prefix + matching_triggers = self._get_matching_triggers(event_type, payload) + + if not matching_triggers: + # Create an TriggerEvent record even without matching active trigger definition + trigger_event = TriggerEvent.objects.create( + trigger=None, + source=event_source, + event_time=event_time, + event_type=event_type, + event_headers=headers, + event_body=payload, + ) + trigger_event.save() + logger.info("Trigger event recorded with id %s", trigger_event.id) + logger.info("No matching active trigger found") + + for trigger in matching_triggers: + # Create a TriggerEvent record and start a pipeline execution + trigger_event = TriggerEvent.objects.create( + trigger=trigger, + source=event_source, + event_time=event_time, + event_type=event_type, + event_headers=headers, + event_body=payload, + ) + logger.info("Trigger event created with id %s", trigger_event.id) + if user is None: + user = self._get_matching_user(trigger.owner) + logger.info("Will execute a pipeline for user %s", user) + try: + pipeline_id = trigger.pipeline.id + params = trigger.params_default + pipeline_run = PipelineRunViewSet._create(user, pipeline_id, params) + logger.debug("Pipeline run: %s", pipeline_run) + trigger_event.pipeline_run = pipeline_run + except Exception as exception: + logger.error("Exception: %s", exception) + trigger_event.save() + # --- + + # if event_type == "org.eoepca.webhook.github.ping": + # logger.debug("Received a ping event from GitHub") + # logger.debug("Originating GitHub repository: %s", event_source) + + # elif event_type == "org.eoepca.webhook.gitlab.ping": + # logger.debug("Received a ping event from GitLab") + # logger.debug("Originating GitLab repository: %s", event_source) if event_type.endswith(".probes.health"): logger.debug("Received a health check event") @@ -111,30 +225,30 @@ def post(self, request, *args, **kwargs): "type": RESPONSE_TYPE_PREFIX + ".ok" }) - if user and pipeline_id and event_type.endswith(".event.pipeline.execute"): - pipeline_run = PipelineRunViewSet._create(user, pipeline_id, payload) - logger.debug("Pipeline run: %s", pipeline_run) - - res_data.update({ - "status": "accepted", - # "processed_id": event_id, - "timestamp": int(time.time()), - "message": "Pipeline execution created.", - }) - res_attrs.update({ - "type": RESPONSE_TYPE_PREFIX + ".accepted", - "subject": pipeline_run.id, - }) - - res_body, res_headers = encode(res_attrs, res_data) + # if user and pipeline_id and event_type.endswith(".event.pipeline.execute"): + # pipeline_run = PipelineRunViewSet._create(user, pipeline_id, payload) + # logger.debug("Pipeline run: %s", pipeline_run) + + # res_data.update({ + # "status": "accepted", + # # "processed_id": event_id, + # "timestamp": int(time.time()), + # "message": "Pipeline execution created.", + # }) + # res_attrs.update({ + # "type": RESPONSE_TYPE_PREFIX + ".accepted", + # "subject": pipeline_run.id, + # }) + + res_body, res_headers = ce_encode(res_attrs, res_data) logger.debug("Response Headers: %s", res_headers) logger.debug("Response Body: %s", res_body) return Response(res_body, status=202, headers=res_headers) - except Exception as e: - logger.error("Exception: %s", e) + except Exception as exception: + logger.error("Exception: %s", exception) return Response( - {"error": str(e)}, + {"error": str(exception)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR ) @@ -160,13 +274,13 @@ def get_queryset(self): def perform_create(self, serializer): try: serializer.save(owner=self.request.user, template=pipeline_cwl_template) - except IntegrityError as ie: - logger.error(ie) + except IntegrityError as error: + logger.error(error) raise ValidationError( { "detail": "A pipeline with this name, version, and owner already exists." } - ) from ie + ) from error def get_permissions(self): if self.action in ["create", "list", "retrieve"]: @@ -179,16 +293,18 @@ def get_permissions(self): class PipelineRunViewSet(viewsets.ModelViewSet): serializer_class = serializers.PipelineRunSerializer permission_classes = [permissions.IsAuthenticated] + lookup_url_kwarg = "run_id" + lookup_field = "id" def get_queryset(self): user = self.request.user logger.info("User %s is requesting pipeline runs (admin=%s)", user, user.is_staff) p_id = self.kwargs["pipeline_id"] if user.is_staff: - if p_id == "_": + if p_id in ["_", "-"]: return PipelineRun.objects return PipelineRun.objects.filter(pipeline_id=p_id) - if p_id == "_": + if p_id in ["_", "-"]: return PipelineRun.objects.filter(started_by=self.request.user) return PipelineRun.objects.filter(pipeline_id=p_id, started_by=self.request.user) @@ -209,6 +325,7 @@ def _create(user: User, pipeline_id: str, data: dict) -> PipelineRun: pipeline_run = PipelineRun.objects.create( pipeline=pipeline, usage_report="", + digest="", start_time=timezone.now(), status="starting", started_by=user, @@ -251,6 +368,42 @@ def create(self, request, *args, **kwargs) -> Response: status=status.HTTP_201_CREATED ) + # Customise how pipeline runs are updated using PATCH + def partial_update(self, request, *args, **kwargs) -> Response: + user = request.user + if not user or not user.is_staff: + raise PermissionDenied("You must be an administrator to manually update digest quality.") + logger.info("User %s is updating a pipeline run (admin=%s)", user, user.is_staff) + pipeline_run = self.get_object() + data = request.data or {} + logger.debug("Patching run %s with data: %s", pipeline_run, data) + if data.get("digest", {}).get("digest_quality", None): + new_quality = data["digest"]["digest_quality"] + logger.debug("Manually changing the digest quality status to: %s", data) + current_digest = pipeline_run.digest + updated_digest = dict(current_digest) + updated_digest["manual"] = { + "digest_quality": new_quality, + "time": timezone.now().isoformat(timespec="seconds"), + "user": user.username, + } + logger.debug("Updated digest: %s", updated_digest) + request.data["digest"] = updated_digest + try: + # Update the quality status in GitHub/GitLab/... + update_quality_status(pipeline_run, new_quality) + except Exception as error: + logger.error(error) + raise ValidationError( + { "detail": "Failed to update the status in the git repository." } + ) from error + return super().partial_update(request, *args, **kwargs) + # Only the digest quality status may be updated. In all other cases, raise a BAD REQUEST + return Response( + { "error": "Invalid pipeline run update request" }, + status=status.HTTP_400_BAD_REQUEST + ) + @staticmethod def _merge_params(subworkflow: Subworkflow, default_inputs: dict) -> dict: if subworkflow.slug not in default_inputs: @@ -259,13 +412,24 @@ def _merge_params(subworkflow: Subworkflow, default_inputs: dict) -> dict: merged_params = subworkflow.user_params.copy() defaults = default_inputs[subworkflow.slug] + # for key, value in merged_params.items(): + # if isinstance(value, dict) and key in defaults: + # for sub_key, sub_value in value.items(): + # if isinstance(sub_value, dict) and "default" in sub_value: + # if sub_key in defaults[key] and "default" in defaults[key][sub_key]: + # merged_params[key][sub_key]["default"] = defaults[key][sub_key]["default"] for key, value in merged_params.items(): - if isinstance(value, dict) and key in defaults: - for sub_key, sub_value in value.items(): - if isinstance(sub_value, dict) and "default" in sub_value: - if sub_key in defaults[key] and "default" in defaults[key][sub_key]: - merged_params[key][sub_key]["default"] = defaults[key][sub_key]["default"] - + if not isinstance(value, dict): + continue + if key not in defaults: + continue + for sub_key, sub_value in value.items(): + if not isinstance(sub_value, dict): + continue + if "default" not in sub_value: + continue + if sub_key in defaults[key] and "default" in defaults[key][sub_key]: + merged_params[key][sub_key]["default"] = defaults[key][sub_key]["default"] return merged_params @staticmethod @@ -277,10 +441,11 @@ def _render_cwl(pipeline): logger.debug("Rendering subworkflow '%s'", subworkflow) subtemplate = Template(subworkflow.definition) subcontext = {"tools": list(subworkflow.tools.all())} + defaults = pipeline.default_inputs subtool = { "definition": subtemplate.render(subcontext), "slug": subworkflow.pk, - "user_params": PipelineRunViewSet._merge_params(subworkflow, pipeline.default_inputs), + "user_params": PipelineRunViewSet._merge_params(subworkflow, defaults), "pipeline_step": subworkflow.pipeline_step, } rendered_subworkflows.append(subtool) @@ -326,6 +491,9 @@ def create(self, request, *args, **kwargs): # The (optional) instance parameter allows to distinguish reports from scattered steps instance = request.query_params.get("instance", "") + # The (optional) instance parameter allows to distinguish reports from scattered steps + instance = request.query_params.get("instance", "") + try: run = PipelineRun.objects.get(pipeline__id=pipeline_id, id=run_id) except PipelineRun.DoesNotExist: @@ -341,7 +509,7 @@ def create(self, request, *args, **kwargs): if JobReport.objects.filter(run=run, name=tool_name, instance=instance).exists(): logger.warning( - "Could not create a job report: A job report for '%s'/'%s' already exists in run %s", + "Could not create a job report: A report for '%s'/'%s' already exists in run %s", tool_name, instance, run_id @@ -351,11 +519,26 @@ def create(self, request, *args, **kwargs): status=status.HTTP_400_BAD_REQUEST, ) + if request.FILES: + logger.info("The request contains files: %s", ", ".join(request.FILES.keys())) + report_file = request.FILES.get("report") + report_data = json.loads(report_file.read().decode("utf-8")) + digest_file = request.FILES.get("digest") + digest_data = json.loads(digest_file.read().decode("utf-8")) + #logger.info("Report: %s", json.dumps(report_data, indent=2)) + logger.info("Report digest: %s", json.dumps(digest_data, indent=2)) + logger.info("Successfully read both files from the request") + else: + logger.info("No FILES found in the request") + report_data = request.data + digest_data = {} + job_report = JobReport.objects.create( name=tool_name, instance=instance, run=run, - output=request.data, + output=report_data, + digest=digest_data, created_at=timezone.now() ) @@ -374,11 +557,11 @@ class SubworkflowViewSet(viewsets.ReadOnlyModelViewSet): def get_queryset(self): user = self.request.user if user: - logger.info(f"User {user} is requesting the tools information") + logger.info("User %s is requesting the tools information", user) else: - logger.info(f"Anonymous user is requesting the tools information") + logger.info("Anonymous user is requesting the tools information") if user and user.is_staff: - logger.info(f"User {user} is staff / admin") + logger.info("User %s is staff / admin", user) # Admins may get all the tools, whatever their status or availability flag return Subworkflow.objects.all() return Subworkflow.objects.filter(available=True) @@ -386,4 +569,128 @@ def get_queryset(self): class TagViewSet(viewsets.ReadOnlyModelViewSet): queryset = Tag.objects.all() - serializer_class = serializers.TagSerializer \ No newline at end of file + serializer_class = serializers.TagSerializer + + +class TriggerTypeViewSet(viewsets.ReadOnlyModelViewSet): + serializer_class = serializers.TriggerTypeSerializer + + def get_queryset(self): + user = self.request.user + if user: + logger.info("User %s is requesting the trigger types information", user) + else: + logger.info("Anonymous user is requesting the trigger types information") + if user and user.is_staff: + logger.info("User %s is staff / admin", user) + # Admins may get all the tools, whatever their status or availability flag + return TriggerType.objects.all() + return TriggerType.objects.filter(available=True) + + +class TriggerViewSet(viewsets.ModelViewSet): + serializer_class = serializers.TriggerSerializer + + def get_queryset(self): + if self.request.user.is_staff: + return Trigger.objects.all().annotate(event_count=Count("trigger_events")) + return Trigger.objects.filter( + ~Q(status="Deleted"), + Q(owner=self.request.user) | Q(owner__isnull=True) + ).annotate(event_count=Count("trigger_events")) + + def get_permissions(self): + if self.action in ["create", "list", "retrieve"]: + return [permissions.IsAuthenticated()] + if self.action in ["update", "partial_update", "destroy"]: + return [permissions.IsAuthenticated(), IsOwnerOrAdmin()] + return super().get_permissions() + + # Intercept the POST (create) request + def perform_create(self, serializer): + # Look up owner since validation has already passed in the serializer + requested_owner = serializer.validated_data.get("owner") + if not requested_owner: + # No specific owner requested => Assign the trigger to the current user + serializer.save(owner=self.request.user) + return + if not self.request.user.is_staff: + # Current user is not staff => The trigger may only be assigned to himself + if requested_owner != self.request.user: + raise ValidationError({ + "owner": [f"You may not assign a trigger to user '{requested_owner.name}'."], + }) + serializer.save(owner=requested_owner) + + # Intercept the PUT and PATCH (update) requests + def perform_update(self, serializer): + return self.perform_create(serializer) + + # Intercept the DELETE request + def perform_destroy(self, instance): + # Update the trigger status instead of deleting it. + # Otherwise all the associated events and pipeline runs are deleted as well due to cascading. + logger.info("Disabling and setting the trigger status to 'Deleted' instead of deleting the trigger %s", instance) + instance.status = "Deleted" + instance.enabled = False + instance.save() + + +class TriggerEventViewSet(viewsets.ReadOnlyModelViewSet): + serializer_class = serializers.TriggerEventSerializer + + def get_queryset(self): + trigger_id = self.kwargs["trigger_id"] + user = self.request.user + if user: + logger.info( + "User %s is requesting trigger events information for trigger %s", + user, + trigger_id + ) + else: + logger.info( + "Anonymous user is requesting trigger events information for trigger %s", + trigger_id + ) + if user and user.is_staff: + logger.info("User %s is staff / admin", user) + # Admins may get all the tools, whatever their status or availability flag + if trigger_id in ["_", "-"]: + return TriggerEvent.objects.all() + return TriggerEvent.objects.filter(trigger__slug=trigger_id) + # Return only the trigger events owned by the requesting user + if trigger_id in ["_", "-"]: + return TriggerEvent.objects.filter(pipeline_run__user=user) + return TriggerEvent.objects.filter(pipeline_run__user=user, trigger__slug=trigger_id) + + +class TriggerRunViewSet(viewsets.ReadOnlyModelViewSet): + # This viewset returns serialized pipeline runs + # (same as PipelineRunViewSet but filtered differently) + serializer_class = serializers.PipelineRunSerializer + permission_classes = [permissions.IsAuthenticated] + + @staticmethod + def _get_matching_runs(trigger_events): + logger.debug("Trigger events: %s", trigger_events) + run_ids = [event.pipeline_run_id for event in trigger_events if event.pipeline_run_id] + logger.debug("Pipeline run IDs: %s", run_ids) + pipeline_runs = PipelineRun.objects.filter(id__in=run_ids) + logger.debug("Pipeline runs: %s", pipeline_runs) + return pipeline_runs + + def get_queryset(self): + user = self.request.user + trigger_id = self.kwargs["trigger_id"] + logger.info("User '%s' is requesting pipeline runs for trigger '%s'", user, trigger_id) + trigger_events = None + if user.is_staff: + if trigger_id in ["_", "-"]: + trigger_events = TriggerEvent.objects.all() + else: + trigger_events = TriggerEvent.objects.filter(trigger__slug=trigger_id) + return self._get_matching_runs(trigger_events) + if trigger_id in ["_", "-"]: + return PipelineRun.objects.filter(started_by=self.request.user) + return PipelineRun.objects.filter(pipeline_id=p_id, started_by=self.request.user) diff --git a/backend/pycalrissian/execution.py b/backend/pycalrissian/execution.py index 8f7b905..a5d73fe 100644 --- a/backend/pycalrissian/execution.py +++ b/backend/pycalrissian/execution.py @@ -78,14 +78,14 @@ def is_active(self) -> bool: def get_output(self) -> Dict: """Returns the job output""" - if self.is_succeeded: + if self.is_succeeded(): filename = self.get_file_from_volume(["output.json"])[0] with open(filename, "r") as staged_file: return json.load(staged_file) def get_usage_report(self) -> Dict: """Returns the job usage report""" - if self.is_complete: + if self.is_complete(): try: filename = self.get_file_from_volume(["report.json"])[0] with open(filename, "r") as staged_file: @@ -122,7 +122,7 @@ def get_file_from_volume(self, filenames): def get_log(self): """Returns the job execution log""" - if self.is_complete: + if self.is_complete(): return self._get_container_log(ContainerNames.CALRISSIAN) return None diff --git a/backend/pycalrissian/utils.py b/backend/pycalrissian/utils.py index 8917b8c..c39d0ae 100644 --- a/backend/pycalrissian/utils.py +++ b/backend/pycalrissian/utils.py @@ -64,7 +64,7 @@ def _create_pod(self): ], }, } - resp = self.context.core_v1_api.create_namespaced_pod( + _ = self.context.core_v1_api.create_namespaced_pod( body=pod_manifest, namespace=self.context.namespace, ) @@ -382,4 +382,4 @@ def copy_from_volume( sys.stdout.flush() os.dup2(old_out_fd, old_out.fileno()) sys.stdout = old_out - helper_pod.dismiss() \ No newline at end of file + helper_pod.dismiss() diff --git a/backend/requirements.txt b/backend/requirements.txt index 7ed2219..d111b8c 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -13,4 +13,7 @@ kubernetes==28.1.0 loguru==0.7.3 mozilla-django-oidc==4.0.1 psycopg2==2.9.11 -cloudevents==1.12 \ No newline at end of file +# To encode and decode cloudevents in the views +cloudevents==1.12 +# To apply rules to digest issues +rule-engine==5.0.1 \ No newline at end of file diff --git a/backend/scripts/install_shell_tools.sh b/backend/scripts/install_shell_tools.sh index 944bbb2..f5e2c14 100644 --- a/backend/scripts/install_shell_tools.sh +++ b/backend/scripts/install_shell_tools.sh @@ -1,5 +1,5 @@ #!/usr/bin/sh apt update -apt install -y vim procps +apt install -y vim procps htop rm -rf /var/lib/apt/lists/* diff --git a/backend/test/tests.py b/backend/test/tests.py index 11b527c..2cb820a 100644 --- a/backend/test/tests.py +++ b/backend/test/tests.py @@ -37,14 +37,17 @@ class PipelineViewTest(TestCase): def setUp(self): self.user = User.objects.create_user( username="testuser", - password="pass" + password="pass", # noqa ) self.client = APIClient() self.token = self.get_token() self.client.credentials(HTTP_AUTHORIZATION=f"Bearer {self.token}") def get_token(self): - response = self.client.post("/api/token/",{"username": "testuser", "password": "pass"}) + response = self.client.post( + "/api/token/", + {"username": "testuser", "password": "pass"}, # noqa + ) return response.data["access"] def test_pipeline_create_unauthenticated_fail(self): @@ -97,7 +100,7 @@ class PipelineRunViewTest(TestCase): def setUp(self): self.user = User.objects.create_user( username="runner", - password="pass" + password="pass", # noqa ) self.pipeline = Pipeline.objects.create( name="PipelineTest",