From 0a0b97236d32d171cc187b5b24d510f847962872 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Thu, 21 Jan 2021 18:24:36 +0100
Subject: [PATCH 01/69] Introduce report analyzer

Signed-off-by: Pawel Czarnecki <pczarnecki@antmicro.com>
---
 .github/kokoro/run.sh             |  18 +++
 .github/workflows/analyze.sh      |  18 +++
 .github/workflows/sv-tests-ci.yml |   7 ++
 tools/report_analyzer.py          | 183 ++++++++++++++++++++++++++++++
 4 files changed, 226 insertions(+)
 create mode 100755 .github/workflows/analyze.sh
 create mode 100755 tools/report_analyzer.py

diff --git a/.github/kokoro/run.sh b/.github/kokoro/run.sh
index 13d7039276767..449c1861bda73 100755
--- a/.github/kokoro/run.sh
+++ b/.github/kokoro/run.sh
@@ -78,6 +78,24 @@ echo "----------------------------------------"
 )
 echo "----------------------------------------"
 
+echo
+echo "========================================"
+echo "Running Analysis"
+echo "----------------------------------------"
+{
+	ANALYZER=$GIT_CHECKOUT"/tools/report_analyzer.py"
+	OUT_DIR=$GIT_CHECKOUT"/out/report/"
+	COMPARE_REPORT=$OUT_DIR"/report.csv"
+	BASE_REPORT=$OUT_DIR"/base_report.csv"
+	CHANGES_SUMMARY=$OUT_DIR"/changes_summary.json"
+
+	# Get base report from sv-tests master run
+	wget https://symbiflow.github.io/sv-tests-results/report.csv -O $BASE_REPORT
+
+	python $ANALYZER $COMPARE_REPORT $BASE_REPORT -o $CHANGES_SUMMARY
+}
+echo "----------------------------------------"
+
 if [[ $KOKORO_TYPE = continuous ]]; then
 	#   - "make report USE_ALL_RUNNERS=1"
 	#   - "touch out/report/.nojekyll"
diff --git a/.github/workflows/analyze.sh b/.github/workflows/analyze.sh
new file mode 100755
index 0000000000000..7df3bdcaaec30
--- /dev/null
+++ b/.github/workflows/analyze.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+ANALYZER=$PWD"/tools/report_analyzer.py"
+OUT_DIR=$PWD"/out/report/"
+COMPARE_REPORT=$OUT_DIR"/report.csv"
+BASE_REPORT=$OUT_DIR"/base_report.csv"
+CHANGES_SUMMARY=$OUT_DIR"/changes_summary.json"
+
+set -x
+set -e
+
+# Get base report from sv-tests master run
+wget https://symbiflow.github.io/sv-tests-results/report.csv -O $BASE_REPORT
+
+python $ANALYZER $COMPARE_REPORT $BASE_REPORT -o $CHANGES_SUMMARY
+
+set +e
+set +x
diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index a46b6f592ccbb..d0276167aea58 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -64,3 +64,10 @@ jobs:
       - name: Run
         run:
           ./.github/workflows/run.sh
+      - name: Analyze
+        run:
+          ./.github/workflows/analyze.sh
+      - uses: actions/upload-artifact@v2
+        with:
+          name: changes_summary_${{ matrix.env.JOB_NAME }}.json
+          path: ./out/report/changes_summary.json
diff --git a/tools/report_analyzer.py b/tools/report_analyzer.py
new file mode 100755
index 0000000000000..f4d8f489a1396
--- /dev/null
+++ b/tools/report_analyzer.py
@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2021 The SymbiFlow Authors.
+#
+# Use of this source code is governed by a ISC-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/ISC
+#
+# SPDX-License-Identifier: ISC
+
+import argparse
+import csv
+import json
+import logging
+import sys
+
+relevant_headers = ["Tool", "TestName", "Pass"]
+
+file_handler = logging.FileHandler(filename='analyze.log')
+stdout_handler = logging.StreamHandler(sys.stdout)
+handlers = [file_handler, stdout_handler]
+
+logging.basicConfig(
+    level=logging.DEBUG,
+    format='[%(asctime)s]: %(message)s',
+    handlers=handlers)
+
+logger = logging.getLogger('report_analyzer_logger')
+
+
+def log_list(header, entity_list):
+    logger.debug(header)
+    for elem in entity_list:
+        logger.debug("  - " + elem)
+
+
+def get_data(csv_path):
+    with open(csv_path, newline="") as csv_file:
+        report = list(csv.DictReader(csv_file))
+        header = report[0].keys()
+
+        assert set(relevant_headers).issubset(
+            header), "Lack of crucial headers in CSV report " + csv_path
+
+        tools = set(row["Tool"] for row in report)
+
+        sorted_report = {}
+        for tool in tools:
+            sorted_report[tool] = {}
+
+        for row in report:
+            sorted_report[row["Tool"]][row["TestName"]] = row["Pass"]
+
+    return sorted_report
+
+
+def check_tool(tool_reportA, tool_reportB):
+    results = {
+        "new_passes": [],
+        "new_failures": [],
+        "added": [],
+        "removed": [],
+        "summary": {},
+    }
+
+    testsA = set(tool_reportA.keys())
+    testsB = set(tool_reportB.keys())
+
+    tests_added = testsA.difference(testsB)
+    tests_removed = testsB.difference(testsA)
+
+    tests_comparable = testsA.intersection(testsB)
+
+    added_cnt = len(tests_added)
+    removed_cnt = len(tests_removed)
+    no_change_cnt = 0
+
+    for test in tests_comparable:
+        res = check_test(tool_reportA[test], tool_reportB[test])
+        if (res == -1):
+            results["new_failures"].append(test)
+        elif (res == 1):
+            results["new_passes"].append(test)
+        else:
+            no_change_cnt += 1
+
+    fail_cnt = len(results["new_failures"])
+    pass_cnt = len(results["new_passes"])
+
+    for added_test in tests_added:
+        results["added"].append(added_test)
+
+    for removed_test in tests_removed:
+        results["removed"].append(removed_test)
+
+    results["summary"]["new_failures"] = fail_cnt
+    results["summary"]["new_passes"] = pass_cnt
+    results["summary"]["added"] = added_cnt
+    results["summary"]["removed"] = removed_cnt
+    results["summary"]["not_affected"] = no_change_cnt
+    logger.debug("     - new failures: " + str(fail_cnt))
+    logger.debug("     - new passes: " + str(pass_cnt))
+    logger.debug("     - tests added: " + str(added_cnt))
+    logger.debug("     - tests removed: " + str(removed_cnt))
+    logger.debug("     - tests not affected: " + str(no_change_cnt))
+
+    return results
+
+
+def check_test(test_reportA, test_reportB):
+    if (test_reportA == test_reportB):
+        return 0
+    elif (test_reportA == "True" and test_reportB == "False"):
+        return 1
+    elif (test_reportA == "False" and test_reportB == "True"):
+        return -1
+    else:
+        raise ValueError(
+            "unknown test result occured: A -> " + test_reportA + " B -> " +
+            test_reportB)
+
+
+def check_reports(reportA, reportB):
+    summary = {
+        "comparable_tools": {},
+        "added_tools": [],
+        "removed_tools": [],
+    }
+
+    toolsA = set(reportA.keys())
+    toolsB = set(reportB.keys())
+    tools = toolsA.intersection(toolsB)
+
+    if (toolsA != toolsB):
+        tools_added = toolsA.difference(toolsB)
+        tools_removed = toolsB.difference(toolsA)
+        summary["added_tools"] = list(tools_added)
+        summary["removed_tools"] = list(tools_removed)
+
+    log_list("Added tools:", tools_added)
+    log_list("Removed tools:", tools_removed)
+
+    logger.debug("Comparable tools:")
+    for tool in tools:
+        logger.debug("  - " + tool + ":")
+        tool_results = check_tool(reportA[tool], reportB[tool])
+        summary["comparable_tools"][tool] = tool_results
+
+    return summary
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "report_compare",
+        help="csv report that will be compared with base report")
+    parser.add_argument(
+        "report_base", help="csv report that will be a base of comparison")
+    parser.add_argument(
+        "-o",
+        "--output",
+        dest="output_path",
+        default="report_summary.json",
+        help="path to output json file, defaults to \"report_summary.json\"")
+    parser.add_argument(
+        "-q", "--quiet", action="store_true", help="disable log output")
+    args = parser.parse_args()
+
+    reportA = get_data(args.report_compare)
+    reportB = get_data(args.report_base)
+
+    if (args.quiet):
+        logger.setLevel("INFO")
+
+    summary = check_reports(reportA, reportB)
+
+    with open(args.output_path, "w") as json_file:
+        json.dump(summary, json_file, indent=4)
+
+
+if __name__ == "__main__":
+    main()

From 4ea2c87291b9a79c45e374d81343211ae19836b2 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Thu, 4 Feb 2021 15:03:54 +0100
Subject: [PATCH 02/69] upload also reports

---
 .github/workflows/sv-tests-ci.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index d0276167aea58..f220b1e16aea4 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -69,5 +69,7 @@ jobs:
           ./.github/workflows/analyze.sh
       - uses: actions/upload-artifact@v2
         with:
-          name: changes_summary_${{ matrix.env.JOB_NAME }}.json
-          path: ./out/report/changes_summary.json
+          name: changes_summary_${{ matrix.env.JOB_NAME }}
+          path: |
+            ./out/report/changes_summary.json
+            ./out/report/report.csv

From c82ec398a5df95cb64b3ddf50829e5ea03b26c57 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Thu, 4 Feb 2021 15:13:59 +0100
Subject: [PATCH 03/69] change naming format

---
 .github/workflows/analyze.sh      | 4 +++-
 .github/workflows/sv-tests-ci.yml | 4 ++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/analyze.sh b/.github/workflows/analyze.sh
index 7df3bdcaaec30..e92547154bd67 100755
--- a/.github/workflows/analyze.sh
+++ b/.github/workflows/analyze.sh
@@ -4,7 +4,7 @@ ANALYZER=$PWD"/tools/report_analyzer.py"
 OUT_DIR=$PWD"/out/report/"
 COMPARE_REPORT=$OUT_DIR"/report.csv"
 BASE_REPORT=$OUT_DIR"/base_report.csv"
-CHANGES_SUMMARY=$OUT_DIR"/changes_summary.json"
+CHANGES_SUMMARY=$OUT_DIR"/"$JOB_NAME"_changes_summary.json"
 
 set -x
 set -e
@@ -14,5 +14,7 @@ wget https://symbiflow.github.io/sv-tests-results/report.csv -O $BASE_REPORT
 
 python $ANALYZER $COMPARE_REPORT $BASE_REPORT -o $CHANGES_SUMMARY
 
+mv $COMPARE_REPORT $OUT_DIR"/"$JOB_NAME"_report.csv"
+
 set +e
 set +x
diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index f220b1e16aea4..2de3bd900d396 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -71,5 +71,5 @@ jobs:
         with:
           name: changes_summary_${{ matrix.env.JOB_NAME }}
           path: |
-            ./out/report/changes_summary.json
-            ./out/report/report.csv
+            ./out/report/${{ matrix.env.JOB_NAME }}_changes_summary.json
+            ./out/report/${{ matrix.env.JOB_NAME }}_report.csv

From 8df9ff70294ae97171e0dc2a473e8fab0c291b7c Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Thu, 4 Feb 2021 15:17:42 +0100
Subject: [PATCH 04/69] test on few quick tools

---
 .github/workflows/sv-tests-ci.yml | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 2de3bd900d396..9bd37dfe9d9ae 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -34,20 +34,20 @@ jobs:
       fail-fast: false
       matrix:
         env:
-          - { JOB_NAME: iverilog, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: iverilog, MAKEFLAGS: -j2 }
           - { JOB_NAME: moore, MAKEFLAGS: -j2 }
-          - { JOB_NAME: odin_ii, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: odin_ii, MAKEFLAGS: -j2 }
           - { JOB_NAME: slang, MAKEFLAGS: -j2 }
-          - { JOB_NAME: sv-parser, MAKEFLAGS: -j2 }
-          - { JOB_NAME: surelog, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: sv-parser, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: surelog, MAKEFLAGS: -j2 }
           - { JOB_NAME: tree-sitter-verilog, MAKEFLAGS: -j2 }
           - { JOB_NAME: yosys, MAKEFLAGS: -j2 }
-          - { JOB_NAME: antmicro-yosys-complete, MAKEFLAGS: -j2 }
-          - { JOB_NAME: verible, MAKEFLAGS: -j2 }
-          - { JOB_NAME: verilator, MAKEFLAGS: -j2 }
-          - { JOB_NAME: uhdm-integration-verilator, MAKEFLAGS: -j2 }
-          - { JOB_NAME: uhdm-integration-yosys, MAKEFLAGS: -j2 }
-          - { JOB_NAME: zachjs-sv2v, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: antmicro-yosys-complete, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: verible, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: verilator, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: uhdm-integration-verilator, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: uhdm-integration-yosys, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: zachjs-sv2v, MAKEFLAGS: -j2 }
 
     name: ${{ matrix.env.JOB_NAME }}
     env: ${{ matrix.env }}

From b8212bd207251ac37154e8db128c66493fb83c4b Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Thu, 4 Feb 2021 17:05:35 +0100
Subject: [PATCH 05/69] download artifacts test

---
 .github/workflows/sv-tests-ci.yml | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 9bd37dfe9d9ae..3e63a82d96368 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -73,3 +73,16 @@ jobs:
           path: |
             ./out/report/${{ matrix.env.JOB_NAME }}_changes_summary.json
             ./out/report/${{ matrix.env.JOB_NAME }}_report.csv
+
+  Summary:
+    name: Summary
+    runs-on: ubuntu-18.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+      - uses: actions/download-artifact@v2
+      - name: Display structure of downloaded files
+        run: ls -R

From 689e3b742f0b7171a20aab5040b7aabe4708b1cf Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Thu, 4 Feb 2021 17:06:27 +0100
Subject: [PATCH 06/69] test only 2 tools

---
 .github/workflows/sv-tests-ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 3e63a82d96368..3846616395c22 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -35,13 +35,13 @@ jobs:
       matrix:
         env:
           #- { JOB_NAME: iverilog, MAKEFLAGS: -j2 }
-          - { JOB_NAME: moore, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: moore, MAKEFLAGS: -j2 }
           #- { JOB_NAME: odin_ii, MAKEFLAGS: -j2 }
           - { JOB_NAME: slang, MAKEFLAGS: -j2 }
           #- { JOB_NAME: sv-parser, MAKEFLAGS: -j2 }
           #- { JOB_NAME: surelog, MAKEFLAGS: -j2 }
           - { JOB_NAME: tree-sitter-verilog, MAKEFLAGS: -j2 }
-          - { JOB_NAME: yosys, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: yosys, MAKEFLAGS: -j2 }
           #- { JOB_NAME: antmicro-yosys-complete, MAKEFLAGS: -j2 }
           #- { JOB_NAME: verible, MAKEFLAGS: -j2 }
           #- { JOB_NAME: verilator, MAKEFLAGS: -j2 }

From 3a9e144de126162573478573b71e1411885e8cee Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Thu, 4 Feb 2021 17:12:12 +0100
Subject: [PATCH 07/69] Summary needs Run

---
 .github/workflows/sv-tests-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 3846616395c22..fd92f60e416da 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -77,6 +77,7 @@ jobs:
   Summary:
     name: Summary
     runs-on: ubuntu-18.04
+    needs: Run
     steps:
       - name: Checkout code
         uses: actions/checkout@v2

From 36fbdeef8250a0555e1a8d9a86ae81045e9825fc Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Thu, 4 Feb 2021 18:47:32 +0100
Subject: [PATCH 08/69] test summary script

---
 .github/workflows/summary.sh      | 29 +++++++++++++++++++++++++++++
 .github/workflows/sv-tests-ci.yml | 16 +++++++++++++++-
 2 files changed, 44 insertions(+), 1 deletion(-)
 create mode 100755 .github/workflows/summary.sh

diff --git a/.github/workflows/summary.sh b/.github/workflows/summary.sh
new file mode 100755
index 0000000000000..e8598721cbed0
--- /dev/null
+++ b/.github/workflows/summary.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+ANALYZER=$PWD"/tools/report_analyzer.py"
+OUT_DIR=$PWD"/out/report/"
+COMPARE_REPORT=$OUT_DIR"/tests_report.csv"
+BASE_REPORT=$OUT_DIR"/base_report.csv"
+CHANGES_SUMMARY=$OUT_DIR"/tests_summary.json"
+
+set -x
+set -e
+
+# Get base report from sv-tests master run
+wget https://symbiflow.github.io/sv-tests-results/report.csv -O $BASE_REPORT
+
+# Delete headers from all report.csv
+for file in $(find ./out/changes_summary_* -name "*.csv" -print); do
+	sed -i.backup 1,1d $file
+done
+
+# concatenate test reports
+cat $(find ./out/changes_summary_* -name "*.csv" -print) >> $COMPARE_REPORT
+
+# Insert header at the first line of concatenated report
+sed -i 1i\ $(cat $(find ./out/changes_summary_* -name "*.csv.backup" -print | head -1) | head -1) $COMAPRE_REPORT
+
+python $ANALYZER $COMPARE_REPORT $BASE_REPORT -o $CHANGES_SUMMARY
+
+set +e
+set +x
diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index fd92f60e416da..e7f8157d89f5d 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -84,6 +84,20 @@ jobs:
       - uses: actions/setup-python@v2
         with:
           python-version: 3.7
+      - name: Prepare output directories
+        run: mkdir -p out/report
       - uses: actions/download-artifact@v2
+        with:
+          path: ./out/
       - name: Display structure of downloaded files
-        run: ls -R
+        run: ls -R ./out/
+      - name: Summary
+        run:
+          ./.github/workflows/summary.sh
+      - uses: actions/upload-artifact@v2
+        with:
+          name: tests_summary
+          path: |
+            ./out/report/tests_summary.json
+            ./out/report/tests_report.csv
+            ./out/report/base_report.csv

From 7724d90783901ef7269037ffa5c68df480803ba4 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 08:57:43 +0100
Subject: [PATCH 09/69] typo fix

---
 .github/workflows/summary.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/summary.sh b/.github/workflows/summary.sh
index e8598721cbed0..5f3dc9ac277ca 100755
--- a/.github/workflows/summary.sh
+++ b/.github/workflows/summary.sh
@@ -21,9 +21,10 @@ done
 cat $(find ./out/changes_summary_* -name "*.csv" -print) >> $COMPARE_REPORT
 
 # Insert header at the first line of concatenated report
-sed -i 1i\ $(cat $(find ./out/changes_summary_* -name "*.csv.backup" -print | head -1) | head -1) $COMAPRE_REPORT
+sed -i 1i\ $(cat $(find ./out/changes_summary_* -name "*.csv.backup" -print | head -1) | head -1) $COMPARE_REPORT
 
 python $ANALYZER $COMPARE_REPORT $BASE_REPORT -o $CHANGES_SUMMARY
 
 set +e
 set +x
+

From 1d2036f6962adb3f231ba349ead7924c62f914d9 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 08:58:26 +0100
Subject: [PATCH 10/69] run full test

---
 .github/workflows/sv-tests-ci.yml | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index e7f8157d89f5d..0d8f5a776ff06 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -34,20 +34,20 @@ jobs:
       fail-fast: false
       matrix:
         env:
-          #- { JOB_NAME: iverilog, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: moore, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: odin_ii, MAKEFLAGS: -j2 }
+          - { JOB_NAME: iverilog, MAKEFLAGS: -j2 }
+          - { JOB_NAME: moore, MAKEFLAGS: -j2 }
+          - { JOB_NAME: odin_ii, MAKEFLAGS: -j2 }
           - { JOB_NAME: slang, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: sv-parser, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: surelog, MAKEFLAGS: -j2 }
+          - { JOB_NAME: sv-parser, MAKEFLAGS: -j2 }
+          - { JOB_NAME: surelog, MAKEFLAGS: -j2 }
           - { JOB_NAME: tree-sitter-verilog, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: yosys, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: antmicro-yosys-complete, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: verible, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: verilator, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: uhdm-integration-verilator, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: uhdm-integration-yosys, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: zachjs-sv2v, MAKEFLAGS: -j2 }
+          - { JOB_NAME: yosys, MAKEFLAGS: -j2 }
+          - { JOB_NAME: antmicro-yosys-complete, MAKEFLAGS: -j2 }
+          - { JOB_NAME: verible, MAKEFLAGS: -j2 }
+          - { JOB_NAME: verilator, MAKEFLAGS: -j2 }
+          - { JOB_NAME: uhdm-integration-verilator, MAKEFLAGS: -j2 }
+          - { JOB_NAME: uhdm-integration-yosys, MAKEFLAGS: -j2 }
+          - { JOB_NAME: zachjs-sv2v, MAKEFLAGS: -j2 }
 
     name: ${{ matrix.env.JOB_NAME }}
     env: ${{ matrix.env }}

From 7ce5e4cf00f662e561fd517b287d76d049a943b5 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 12:07:59 +0100
Subject: [PATCH 11/69] fix reference before assignment

---
 tools/report_analyzer.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/tools/report_analyzer.py b/tools/report_analyzer.py
index f4d8f489a1396..3ddbd2bcf2df4 100755
--- a/tools/report_analyzer.py
+++ b/tools/report_analyzer.py
@@ -137,9 +137,8 @@ def check_reports(reportA, reportB):
         tools_removed = toolsB.difference(toolsA)
         summary["added_tools"] = list(tools_added)
         summary["removed_tools"] = list(tools_removed)
-
-    log_list("Added tools:", tools_added)
-    log_list("Removed tools:", tools_removed)
+        log_list("Added tools:", tools_added)
+        log_list("Removed tools:", tools_removed)
 
     logger.debug("Comparable tools:")
     for tool in tools:

From 9ef5a1572e1687d68291c956c5a1c41fd4e949aa Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 14:10:31 +0100
Subject: [PATCH 12/69] test artifacts cleanup

---
 .github/workflows/sv-tests-ci.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 0d8f5a776ff06..09ea7865a34b6 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -101,3 +101,6 @@ jobs:
             ./out/report/tests_summary.json
             ./out/report/tests_report.csv
             ./out/report/base_report.csv
+      - uses: geekyeggo/delete-artifact@v1
+        with:
+          name: changes_summary*

From 791f2b657e63447c66c5e8e567d9f5390c337544 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 15:29:27 +0100
Subject: [PATCH 13/69] Revert "test artifacts cleanup"

This reverts commit 9ef5a1572e1687d68291c956c5a1c41fd4e949aa.
---
 .github/workflows/sv-tests-ci.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 09ea7865a34b6..0d8f5a776ff06 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -101,6 +101,3 @@ jobs:
             ./out/report/tests_summary.json
             ./out/report/tests_report.csv
             ./out/report/base_report.csv
-      - uses: geekyeggo/delete-artifact@v1
-        with:
-          name: changes_summary*

From 7136156d609d57e299f9c3d612e496ac9faa6acd Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 16:26:13 +0100
Subject: [PATCH 14/69] output format experiments

---
 tools/report_analyzer.py | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/tools/report_analyzer.py b/tools/report_analyzer.py
index 3ddbd2bcf2df4..ccdb054f3336a 100755
--- a/tools/report_analyzer.py
+++ b/tools/report_analyzer.py
@@ -23,7 +23,7 @@
 
 logging.basicConfig(
     level=logging.DEBUG,
-    format='[%(asctime)s]: %(message)s',
+    format='%(message)s',
     handlers=handlers)
 
 logger = logging.getLogger('report_analyzer_logger')
@@ -55,7 +55,7 @@ def get_data(csv_path):
     return sorted_report
 
 
-def check_tool(tool_reportA, tool_reportB):
+def check_tool(tool_reportA, tool_reportB, tool_name):
     results = {
         "new_passes": [],
         "new_failures": [],
@@ -99,11 +99,16 @@ def check_tool(tool_reportA, tool_reportB):
     results["summary"]["added"] = added_cnt
     results["summary"]["removed"] = removed_cnt
     results["summary"]["not_affected"] = no_change_cnt
-    logger.debug("     - new failures: " + str(fail_cnt))
-    logger.debug("     - new passes: " + str(pass_cnt))
-    logger.debug("     - tests added: " + str(added_cnt))
-    logger.debug("     - tests removed: " + str(removed_cnt))
-    logger.debug("     - tests not affected: " + str(no_change_cnt))
+    if (fail_cnt | pass_cnt | added_cnt | removed_cnt):
+        logger.debug("  - " + tool_name + ":")
+    if (fail_cnt):
+        logger.debug("     - new failures: " + str(fail_cnt))
+    if (pass_cnt):
+        logger.debug("     - new passes: " + str(pass_cnt))
+    if (added_cnt):
+        logger.debug("     - tests added: " + str(added_cnt))
+    if (removed_cnt):
+        logger.debug("     - tests removed: " + str(removed_cnt))
 
     return results
 
@@ -140,10 +145,9 @@ def check_reports(reportA, reportB):
         log_list("Added tools:", tools_added)
         log_list("Removed tools:", tools_removed)
 
-    logger.debug("Comparable tools:")
+    logger.debug("Compared tools:")
     for tool in tools:
-        logger.debug("  - " + tool + ":")
-        tool_results = check_tool(reportA[tool], reportB[tool])
+        tool_results = check_tool(reportA[tool], reportB[tool], tool)
         summary["comparable_tools"][tool] = tool_results
 
     return summary

From 928754aaed54e35a5ae1fda3738691ee9a342c5b Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 16:57:14 +0100
Subject: [PATCH 15/69] output markdown table summary

---
 conf/requirements.txt    |  1 +
 tools/report_analyzer.py | 29 ++++++++++++++++++++++++++---
 2 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/conf/requirements.txt b/conf/requirements.txt
index 31e0ca3c2fdca..3e758b6c5b5e7 100644
--- a/conf/requirements.txt
+++ b/conf/requirements.txt
@@ -6,4 +6,5 @@ yapf==0.30.0
 psutil
 mako
 make_var
+pytablewriter
 git+https://github.com/lowRISC/fusesoc.git@ot
diff --git a/tools/report_analyzer.py b/tools/report_analyzer.py
index ccdb054f3336a..f6388e5dc71c3 100755
--- a/tools/report_analyzer.py
+++ b/tools/report_analyzer.py
@@ -14,6 +14,7 @@
 import json
 import logging
 import sys
+import pytablewriter
 
 relevant_headers = ["Tool", "TestName", "Pass"]
 
@@ -22,9 +23,7 @@
 handlers = [file_handler, stdout_handler]
 
 logging.basicConfig(
-    level=logging.DEBUG,
-    format='%(message)s',
-    handlers=handlers)
+    level=logging.DEBUG, format='%(message)s', handlers=handlers)
 
 logger = logging.getLogger('report_analyzer_logger')
 
@@ -99,6 +98,7 @@ def check_tool(tool_reportA, tool_reportB, tool_name):
     results["summary"]["added"] = added_cnt
     results["summary"]["removed"] = removed_cnt
     results["summary"]["not_affected"] = no_change_cnt
+
     if (fail_cnt | pass_cnt | added_cnt | removed_cnt):
         logger.debug("  - " + tool_name + ":")
     if (fail_cnt):
@@ -153,6 +153,27 @@ def check_reports(reportA, reportB):
     return summary
 
 
+def prepare_comment(summary):
+    tools = list(summary["comparable_tools"].keys())
+    cols = list(summary["comparable_tools"][tools[0]]["summary"].keys())
+    cols.insert(0, "tool")
+
+    matrix = []
+    for tool in tools:
+        vals = list(summary["comparable_tools"][tool]["summary"].values())
+        vals.insert(0, tool)
+        matrix.append(vals)
+
+    writer = pytablewriter.MarkdownTableWriter()
+    writer.table_name = "Compared test results"
+    writer.header_list = cols
+    writer.value_matrix = matrix
+    writer.write_table()
+    with open("summary.md", "w") as f:
+        writer.stream = f
+        writer.write_table()
+
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument(
@@ -178,6 +199,8 @@ def main():
 
     summary = check_reports(reportA, reportB)
 
+    prepare_comment(summary)
+
     with open(args.output_path, "w") as json_file:
         json.dump(summary, json_file, indent=4)
 

From 1e500e88890692100fec5c885348c99d78fde66c Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 17:02:26 +0100
Subject: [PATCH 16/69] remove logging feature

---
 tools/report_analyzer.py | 35 -----------------------------------
 1 file changed, 35 deletions(-)

diff --git a/tools/report_analyzer.py b/tools/report_analyzer.py
index f6388e5dc71c3..c60a0d01da3e2 100755
--- a/tools/report_analyzer.py
+++ b/tools/report_analyzer.py
@@ -12,27 +12,11 @@
 import argparse
 import csv
 import json
-import logging
 import sys
 import pytablewriter
 
 relevant_headers = ["Tool", "TestName", "Pass"]
 
-file_handler = logging.FileHandler(filename='analyze.log')
-stdout_handler = logging.StreamHandler(sys.stdout)
-handlers = [file_handler, stdout_handler]
-
-logging.basicConfig(
-    level=logging.DEBUG, format='%(message)s', handlers=handlers)
-
-logger = logging.getLogger('report_analyzer_logger')
-
-
-def log_list(header, entity_list):
-    logger.debug(header)
-    for elem in entity_list:
-        logger.debug("  - " + elem)
-
 
 def get_data(csv_path):
     with open(csv_path, newline="") as csv_file:
@@ -99,17 +83,6 @@ def check_tool(tool_reportA, tool_reportB, tool_name):
     results["summary"]["removed"] = removed_cnt
     results["summary"]["not_affected"] = no_change_cnt
 
-    if (fail_cnt | pass_cnt | added_cnt | removed_cnt):
-        logger.debug("  - " + tool_name + ":")
-    if (fail_cnt):
-        logger.debug("     - new failures: " + str(fail_cnt))
-    if (pass_cnt):
-        logger.debug("     - new passes: " + str(pass_cnt))
-    if (added_cnt):
-        logger.debug("     - tests added: " + str(added_cnt))
-    if (removed_cnt):
-        logger.debug("     - tests removed: " + str(removed_cnt))
-
     return results
 
 
@@ -142,10 +115,7 @@ def check_reports(reportA, reportB):
         tools_removed = toolsB.difference(toolsA)
         summary["added_tools"] = list(tools_added)
         summary["removed_tools"] = list(tools_removed)
-        log_list("Added tools:", tools_added)
-        log_list("Removed tools:", tools_removed)
 
-    logger.debug("Compared tools:")
     for tool in tools:
         tool_results = check_tool(reportA[tool], reportB[tool], tool)
         summary["comparable_tools"][tool] = tool_results
@@ -187,16 +157,11 @@ def main():
         dest="output_path",
         default="report_summary.json",
         help="path to output json file, defaults to \"report_summary.json\"")
-    parser.add_argument(
-        "-q", "--quiet", action="store_true", help="disable log output")
     args = parser.parse_args()
 
     reportA = get_data(args.report_compare)
     reportB = get_data(args.report_base)
 
-    if (args.quiet):
-        logger.setLevel("INFO")
-
     summary = check_reports(reportA, reportB)
 
     prepare_comment(summary)

From a4e9f86a903648de31c801b959648322a96011e3 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 17:05:59 +0100
Subject: [PATCH 17/69] variable md path

---
 tools/report_analyzer.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/tools/report_analyzer.py b/tools/report_analyzer.py
index c60a0d01da3e2..61f7d7b544fcb 100755
--- a/tools/report_analyzer.py
+++ b/tools/report_analyzer.py
@@ -123,7 +123,7 @@ def check_reports(reportA, reportB):
     return summary
 
 
-def prepare_comment(summary):
+def prepare_comment(summary, table_path):
     tools = list(summary["comparable_tools"].keys())
     cols = list(summary["comparable_tools"][tools[0]]["summary"].keys())
     cols.insert(0, "tool")
@@ -139,7 +139,7 @@ def prepare_comment(summary):
     writer.header_list = cols
     writer.value_matrix = matrix
     writer.write_table()
-    with open("summary.md", "w") as f:
+    with open(table_path, "w") as f:
         writer.stream = f
         writer.write_table()
 
@@ -157,6 +157,12 @@ def main():
         dest="output_path",
         default="report_summary.json",
         help="path to output json file, defaults to \"report_summary.json\"")
+    parser.add_argument(
+        "-t",
+        "--table",
+        dest="table_path",
+        default="report_summary.md",
+        help="path to output md file with summary, defaults to \"report_summary.md\"")
     args = parser.parse_args()
 
     reportA = get_data(args.report_compare)
@@ -164,7 +170,7 @@ def main():
 
     summary = check_reports(reportA, reportB)
 
-    prepare_comment(summary)
+    prepare_comment(summary, args.table_path)
 
     with open(args.output_path, "w") as json_file:
         json.dump(summary, json_file, indent=4)

From 61e33aa339918f7083f800f5b3675c04d30fd803 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 17:09:03 +0100
Subject: [PATCH 18/69] pass md table

---
 .github/workflows/summary.sh      | 5 +++--
 .github/workflows/sv-tests-ci.yml | 1 +
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/summary.sh b/.github/workflows/summary.sh
index 5f3dc9ac277ca..72b8e2f13a4c0 100755
--- a/.github/workflows/summary.sh
+++ b/.github/workflows/summary.sh
@@ -4,7 +4,8 @@ ANALYZER=$PWD"/tools/report_analyzer.py"
 OUT_DIR=$PWD"/out/report/"
 COMPARE_REPORT=$OUT_DIR"/tests_report.csv"
 BASE_REPORT=$OUT_DIR"/base_report.csv"
-CHANGES_SUMMARY=$OUT_DIR"/tests_summary.json"
+CHANGES_SUMMARY_JSON=$OUT_DIR"/tests_summary.json"
+CHANGES_SUMMARY_MD=$OUT_DIR"/tests_summary.md"
 
 set -x
 set -e
@@ -23,7 +24,7 @@ cat $(find ./out/changes_summary_* -name "*.csv" -print) >> $COMPARE_REPORT
 # Insert header at the first line of concatenated report
 sed -i 1i\ $(cat $(find ./out/changes_summary_* -name "*.csv.backup" -print | head -1) | head -1) $COMPARE_REPORT
 
-python $ANALYZER $COMPARE_REPORT $BASE_REPORT -o $CHANGES_SUMMARY
+python $ANALYZER $COMPARE_REPORT $BASE_REPORT -o $CHANGES_SUMMARY_JSON -t $CHANGES_SUMMARY_MD
 
 set +e
 set +x
diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 0d8f5a776ff06..db7aa27254535 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -99,5 +99,6 @@ jobs:
           name: tests_summary
           path: |
             ./out/report/tests_summary.json
+            ./out/report/tests_summary.md
             ./out/report/tests_report.csv
             ./out/report/base_report.csv

From cb221e51523042bc7de86826ba97ac6d34c9c17f Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 10:37:49 +0100
Subject: [PATCH 19/69] introduce commenter action

---
 .github/workflows/sv-tests-ci.yml | 21 +++++++++++++++++++++
 .github/workflows/template.md     |  1 +
 2 files changed, 22 insertions(+)
 create mode 100644 .github/workflows/template.md

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index db7aa27254535..dce88940b84a9 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -102,3 +102,24 @@ jobs:
             ./out/report/tests_summary.md
             ./out/report/tests_report.csv
             ./out/report/base_report.csv
+
+  Comment:
+    name: Comment
+    runs-on: ubuntu-18.04
+    needs: Summary
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Prepare output directories
+        run: mkdir -p out/report
+      - uses: actions/download-artifact@v2
+        with:
+          name: tests_summary
+          path: ./out/
+      - name: Display structure of downloaded files
+        run: ls -R ./out/
+      - uses: harupy/comment-on-pr@master
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          filename: template.md
diff --git a/.github/workflows/template.md b/.github/workflows/template.md
new file mode 100644
index 0000000000000..a042389697364
--- /dev/null
+++ b/.github/workflows/template.md
@@ -0,0 +1 @@
+hello world!

From d2019ebdd5dfacd9ce55f996cbf4aedafb1f20f0 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 10:39:54 +0100
Subject: [PATCH 20/69] test only 2 packages

---
 .github/workflows/sv-tests-ci.yml | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index dce88940b84a9..c79ae2a2cebd8 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -34,20 +34,20 @@ jobs:
       fail-fast: false
       matrix:
         env:
-          - { JOB_NAME: iverilog, MAKEFLAGS: -j2 }
-          - { JOB_NAME: moore, MAKEFLAGS: -j2 }
-          - { JOB_NAME: odin_ii, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: iverilog, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: moore, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: odin_ii, MAKEFLAGS: -j2 }
           - { JOB_NAME: slang, MAKEFLAGS: -j2 }
-          - { JOB_NAME: sv-parser, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: sv-parser, MAKEFLAGS: -j2 }
           - { JOB_NAME: surelog, MAKEFLAGS: -j2 }
-          - { JOB_NAME: tree-sitter-verilog, MAKEFLAGS: -j2 }
-          - { JOB_NAME: yosys, MAKEFLAGS: -j2 }
-          - { JOB_NAME: antmicro-yosys-complete, MAKEFLAGS: -j2 }
-          - { JOB_NAME: verible, MAKEFLAGS: -j2 }
-          - { JOB_NAME: verilator, MAKEFLAGS: -j2 }
-          - { JOB_NAME: uhdm-integration-verilator, MAKEFLAGS: -j2 }
-          - { JOB_NAME: uhdm-integration-yosys, MAKEFLAGS: -j2 }
-          - { JOB_NAME: zachjs-sv2v, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: tree-sitter-verilog, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: yosys, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: antmicro-yosys-complete, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: verible, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: verilator, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: uhdm-integration-verilator, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: uhdm-integration-yosys, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: zachjs-sv2v, MAKEFLAGS: -j2 }
 
     name: ${{ matrix.env.JOB_NAME }}
     env: ${{ matrix.env }}

From 65ee68846dee87cbe3caf9e31b2dac0d6601a35b Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 10:40:19 +0100
Subject: [PATCH 21/69] no deps at this time

---
 .github/workflows/sv-tests-ci.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index c79ae2a2cebd8..fc63749faa454 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -106,7 +106,6 @@ jobs:
   Comment:
     name: Comment
     runs-on: ubuntu-18.04
-    needs: Summary
     steps:
       - name: Checkout code
         uses: actions/checkout@v2

From f121f4b1037bc66215d4c3a1d3b86ce952434db1 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 10:42:24 +0100
Subject: [PATCH 22/69] remove unnecessary piece of code for testing

---
 .github/workflows/sv-tests-ci.yml | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index fc63749faa454..0a43512043473 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -109,14 +109,6 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
-      - name: Prepare output directories
-        run: mkdir -p out/report
-      - uses: actions/download-artifact@v2
-        with:
-          name: tests_summary
-          path: ./out/
-      - name: Display structure of downloaded files
-        run: ls -R ./out/
       - uses: harupy/comment-on-pr@master
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

From 681d8069310232bd6726d6565698a9e86e1051fd Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 10:53:02 +0100
Subject: [PATCH 23/69] trigger on pr

---
 .github/workflows/sv-tests-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 0a43512043473..d25c079141ac4 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -104,6 +104,7 @@ jobs:
             ./out/report/base_report.csv
 
   Comment:
+    if: github.event_name == 'pull_request'
     name: Comment
     runs-on: ubuntu-18.04
     steps:

From 4090ba9bf1e7b13eb444f4baf8b7d274cbc1cd0e Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 12:12:51 +0100
Subject: [PATCH 24/69] Revert "remove unnecessary piece of code for testing"

This reverts commit 4555aab76e641c9dca5e5f0b3d0d1a167d6f76d5.
---
 .github/workflows/sv-tests-ci.yml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index d25c079141ac4..59002c44ebba1 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -110,6 +110,14 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v2
+      - name: Prepare output directories
+        run: mkdir -p out/report
+      - uses: actions/download-artifact@v2
+        with:
+          name: tests_summary
+          path: ./out/
+      - name: Display structure of downloaded files
+        run: ls -R ./out/
       - uses: harupy/comment-on-pr@master
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

From 8edce46026ee605a2a97346a0547b72482e77a65 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 12:13:52 +0100
Subject: [PATCH 25/69] Revert "no deps at this time"

This reverts commit 811cd464729ce9c7fe2aebc23e22fced5dfc67b2.
---
 .github/workflows/sv-tests-ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 59002c44ebba1..ca653252c5255 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -107,6 +107,7 @@ jobs:
     if: github.event_name == 'pull_request'
     name: Comment
     runs-on: ubuntu-18.04
+    needs: Summary
     steps:
       - name: Checkout code
         uses: actions/checkout@v2

From f4258ffbfdd52ae91bc8c3f8b086bc057d3bb304 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 12:21:32 +0100
Subject: [PATCH 26/69] comment analyzer output

---
 .github/workflows/sv-tests-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index ca653252c5255..556bda34545b5 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -123,4 +123,4 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          filename: template.md
+          filename: ./out/tests_summary/tests_summary.json

From a8ae19445a6aef02bc41828bd3d95dc2211f3d4f Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 13:16:00 +0100
Subject: [PATCH 27/69] fix comment path

---
 .github/workflows/sv-tests-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 556bda34545b5..b9b91279c7847 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -123,4 +123,4 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          filename: ./out/tests_summary/tests_summary.json
+          filename: ./out/tests_summary.json

From 803369c99e3c4e4f0ebe01d41e5411c3ae49c49b Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 13:16:39 +0100
Subject: [PATCH 28/69] test one tool

---
 .github/workflows/sv-tests-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index b9b91279c7847..6aa72b12e5d39 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -39,7 +39,7 @@ jobs:
           #- { JOB_NAME: odin_ii, MAKEFLAGS: -j2 }
           - { JOB_NAME: slang, MAKEFLAGS: -j2 }
           #- { JOB_NAME: sv-parser, MAKEFLAGS: -j2 }
-          - { JOB_NAME: surelog, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: surelog, MAKEFLAGS: -j2 }
           #- { JOB_NAME: tree-sitter-verilog, MAKEFLAGS: -j2 }
           #- { JOB_NAME: yosys, MAKEFLAGS: -j2 }
           #- { JOB_NAME: antmicro-yosys-complete, MAKEFLAGS: -j2 }

From 00bbb3bfcb2cb033036ba3e661224ac1bc532bca Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 13:53:34 +0100
Subject: [PATCH 29/69] fix relative comment file path

---
 .github/workflows/sv-tests-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 6aa72b12e5d39..8db530e40eb65 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -123,4 +123,4 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          filename: ./out/tests_summary.json
+          filename: ../../out/tests_summary.json

From dbdb9c132fe9320af823d1ce251f5d7f1a257575 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 14:10:31 +0100
Subject: [PATCH 30/69] test artifacts cleanup

---
 .github/workflows/sv-tests-ci.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 8db530e40eb65..5cdc5ef12615d 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -124,3 +124,6 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           filename: ../../out/tests_summary.json
+      - uses: geekyeggo/delete-artifact@v1
+        with:
+          name: changes_summary*

From 0379fdb18a1911a85757778109fe76a425e5d4f6 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 14:12:27 +0100
Subject: [PATCH 31/69] Revert "test artifacts cleanup"

This reverts commit f289120cda63607ea94c4970ed32dc273f4b1aec.
---
 .github/workflows/sv-tests-ci.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 5cdc5ef12615d..8db530e40eb65 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -124,6 +124,3 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           filename: ../../out/tests_summary.json
-      - uses: geekyeggo/delete-artifact@v1
-        with:
-          name: changes_summary*

From b0f8c989593a6ecb1cf1f7a009955564a0344669 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 15:34:15 +0100
Subject: [PATCH 32/69] test deleting unuased artifacts

---
 .github/workflows/sv-tests-ci.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 8db530e40eb65..5cdc5ef12615d 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -124,3 +124,6 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           filename: ../../out/tests_summary.json
+      - uses: geekyeggo/delete-artifact@v1
+        with:
+          name: changes_summary*

From b451f458145ced58197c2577093284ecfb1c6888 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 15:37:00 +0100
Subject: [PATCH 33/69] fallback to hello world

---
 .github/workflows/sv-tests-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 5cdc5ef12615d..6ebe8e8fd5ab0 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -123,7 +123,7 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          filename: ../../out/tests_summary.json
+          filename: template.md
       - uses: geekyeggo/delete-artifact@v1
         with:
           name: changes_summary*

From a2900b9a8df0289f72bdcd1f35e3870ef3229778 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 15:50:20 +0100
Subject: [PATCH 34/69] try different artifacts cleanup

---
 .github/workflows/sv-tests-ci.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 6ebe8e8fd5ab0..3a6fcc1352877 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -94,6 +94,10 @@ jobs:
       - name: Summary
         run:
           ./.github/workflows/summary.sh
+      - uses: kolpav/purge-artifacts-action@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          expire-in: 0
       - uses: actions/upload-artifact@v2
         with:
           name: tests_summary
@@ -124,6 +128,3 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           filename: template.md
-      - uses: geekyeggo/delete-artifact@v1
-        with:
-          name: changes_summary*

From e6320540abf9b7a5a201d3c0ea7e7eaf211b8636 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 17:11:54 +0100
Subject: [PATCH 35/69] comment md table on GH PR

---
 .github/workflows/sv-tests-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 3a6fcc1352877..79a5ddb096ad1 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -127,4 +127,4 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          filename: template.md
+          filename: ../../out/report/tests_summary.md

From be3955ed2c7d11be65a985b83994daf31423bf9b Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 17:14:22 +0100
Subject: [PATCH 36/69] don't try to delete artifacts

---
 .github/workflows/sv-tests-ci.yml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 79a5ddb096ad1..518bb1a7547e5 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -94,10 +94,6 @@ jobs:
       - name: Summary
         run:
           ./.github/workflows/summary.sh
-      - uses: kolpav/purge-artifacts-action@v1
-        with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          expire-in: 0
       - uses: actions/upload-artifact@v2
         with:
           name: tests_summary

From 058072af537d0345adefee97964b51ec5acd586f Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 17:51:15 +0100
Subject: [PATCH 37/69] check pytablewriter in environment.yml

---
 conf/environment.yml  | 1 +
 conf/requirements.txt | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/conf/environment.yml b/conf/environment.yml
index a97919e19b2a8..72576729cf2ef 100644
--- a/conf/environment.yml
+++ b/conf/environment.yml
@@ -21,5 +21,6 @@ dependencies:
   - ccache
   - python=3.8
   - pip
+  - pytablewriter
   - pip:                            # Packages installed from PyPI
     - -r file:requirements.txt
diff --git a/conf/requirements.txt b/conf/requirements.txt
index 3e758b6c5b5e7..31e0ca3c2fdca 100644
--- a/conf/requirements.txt
+++ b/conf/requirements.txt
@@ -6,5 +6,4 @@ yapf==0.30.0
 psutil
 mako
 make_var
-pytablewriter
 git+https://github.com/lowRISC/fusesoc.git@ot

From 2f2dd64ab9a4ada9c25beb17e742f5d0d15001f8 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 17:54:23 +0100
Subject: [PATCH 38/69] Revert "check pytablewriter in environment.yml"

This reverts commit 058072af537d0345adefee97964b51ec5acd586f.
---
 conf/environment.yml  | 1 -
 conf/requirements.txt | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/conf/environment.yml b/conf/environment.yml
index 72576729cf2ef..a97919e19b2a8 100644
--- a/conf/environment.yml
+++ b/conf/environment.yml
@@ -21,6 +21,5 @@ dependencies:
   - ccache
   - python=3.8
   - pip
-  - pytablewriter
   - pip:                            # Packages installed from PyPI
     - -r file:requirements.txt
diff --git a/conf/requirements.txt b/conf/requirements.txt
index 31e0ca3c2fdca..3e758b6c5b5e7 100644
--- a/conf/requirements.txt
+++ b/conf/requirements.txt
@@ -6,4 +6,5 @@ yapf==0.30.0
 psutil
 mako
 make_var
+pytablewriter
 git+https://github.com/lowRISC/fusesoc.git@ot

From 5fd7a75f95cd666ce12d51c52126ecc3975e0d8f Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 17:56:12 +0100
Subject: [PATCH 39/69] perform summary inside conda env

---
 .github/workflows/summary.sh | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/.github/workflows/summary.sh b/.github/workflows/summary.sh
index 72b8e2f13a4c0..61d3118b92254 100755
--- a/.github/workflows/summary.sh
+++ b/.github/workflows/summary.sh
@@ -10,6 +10,18 @@ CHANGES_SUMMARY_MD=$OUT_DIR"/tests_summary.md"
 set -x
 set -e
 
+# Get a conda environment
+wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
+bash miniconda.sh -b -p $HOME/miniconda
+source "$HOME/miniconda/etc/profile.d/conda.sh"
+hash -r
+conda config --set always_yes yes --set changeps1 no
+
+conda env create --file conf/environment.yml
+conda activate sv-test-env
+hash -r
+conda info -a
+
 # Get base report from sv-tests master run
 wget https://symbiflow.github.io/sv-tests-results/report.csv -O $BASE_REPORT
 

From 903640a1319f77f80d8656ad95a61c13e13f95f3 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 18:34:56 +0100
Subject: [PATCH 40/69] generate summaries also in conda env

---
 .github/workflows/analyze.sh | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/.github/workflows/analyze.sh b/.github/workflows/analyze.sh
index e92547154bd67..8239a721a57eb 100755
--- a/.github/workflows/analyze.sh
+++ b/.github/workflows/analyze.sh
@@ -9,6 +9,18 @@ CHANGES_SUMMARY=$OUT_DIR"/"$JOB_NAME"_changes_summary.json"
 set -x
 set -e
 
+# Get a conda environment
+wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
+bash miniconda.sh -b -p $HOME/miniconda
+source "$HOME/miniconda/etc/profile.d/conda.sh"
+hash -r
+conda config --set always_yes yes --set changeps1 no
+
+conda env create --file conf/environment.yml
+conda activate sv-test-env
+hash -r
+conda info -a
+
 # Get base report from sv-tests master run
 wget https://symbiflow.github.io/sv-tests-results/report.csv -O $BASE_REPORT
 

From 82b8cbf3127a6740906f47b47428548c38b55cd9 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 19:42:26 +0100
Subject: [PATCH 41/69] only activate conda env

---
 .github/workflows/analyze.sh | 7 -------
 .github/workflows/summary.sh | 7 -------
 2 files changed, 14 deletions(-)

diff --git a/.github/workflows/analyze.sh b/.github/workflows/analyze.sh
index 8239a721a57eb..e1e28e3bead13 100755
--- a/.github/workflows/analyze.sh
+++ b/.github/workflows/analyze.sh
@@ -9,17 +9,10 @@ CHANGES_SUMMARY=$OUT_DIR"/"$JOB_NAME"_changes_summary.json"
 set -x
 set -e
 
-# Get a conda environment
-wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
-bash miniconda.sh -b -p $HOME/miniconda
 source "$HOME/miniconda/etc/profile.d/conda.sh"
 hash -r
-conda config --set always_yes yes --set changeps1 no
-
-conda env create --file conf/environment.yml
 conda activate sv-test-env
 hash -r
-conda info -a
 
 # Get base report from sv-tests master run
 wget https://symbiflow.github.io/sv-tests-results/report.csv -O $BASE_REPORT
diff --git a/.github/workflows/summary.sh b/.github/workflows/summary.sh
index 61d3118b92254..03d03f9193baa 100755
--- a/.github/workflows/summary.sh
+++ b/.github/workflows/summary.sh
@@ -10,17 +10,10 @@ CHANGES_SUMMARY_MD=$OUT_DIR"/tests_summary.md"
 set -x
 set -e
 
-# Get a conda environment
-wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
-bash miniconda.sh -b -p $HOME/miniconda
 source "$HOME/miniconda/etc/profile.d/conda.sh"
 hash -r
-conda config --set always_yes yes --set changeps1 no
-
-conda env create --file conf/environment.yml
 conda activate sv-test-env
 hash -r
-conda info -a
 
 # Get base report from sv-tests master run
 wget https://symbiflow.github.io/sv-tests-results/report.csv -O $BASE_REPORT

From 855fef647175436656c4ef8d5fce8b9c110cc903 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 20:03:15 +0100
Subject: [PATCH 42/69] conda setup is needed in summary job

---
 .github/workflows/summary.sh | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/summary.sh b/.github/workflows/summary.sh
index 03d03f9193baa..61d3118b92254 100755
--- a/.github/workflows/summary.sh
+++ b/.github/workflows/summary.sh
@@ -10,10 +10,17 @@ CHANGES_SUMMARY_MD=$OUT_DIR"/tests_summary.md"
 set -x
 set -e
 
+# Get a conda environment
+wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
+bash miniconda.sh -b -p $HOME/miniconda
 source "$HOME/miniconda/etc/profile.d/conda.sh"
 hash -r
+conda config --set always_yes yes --set changeps1 no
+
+conda env create --file conf/environment.yml
 conda activate sv-test-env
 hash -r
+conda info -a
 
 # Get base report from sv-tests master run
 wget https://symbiflow.github.io/sv-tests-results/report.csv -O $BASE_REPORT

From 058dd3abbccd557974333c0ceb763f300e2a5148 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 20:33:44 +0100
Subject: [PATCH 43/69] download artifact to correct location

---
 .github/workflows/sv-tests-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 518bb1a7547e5..c85f8b37ae837 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -116,7 +116,7 @@ jobs:
       - uses: actions/download-artifact@v2
         with:
           name: tests_summary
-          path: ./out/
+          path: ./out/report
       - name: Display structure of downloaded files
         run: ls -R ./out/
       - uses: harupy/comment-on-pr@master

From 8ae05cef52b11f0d27f89b66c42a0084655f6d79 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 20:34:05 +0100
Subject: [PATCH 44/69] format script

---
 tools/report_analyzer.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tools/report_analyzer.py b/tools/report_analyzer.py
index 61f7d7b544fcb..df2bba7ab2bba 100755
--- a/tools/report_analyzer.py
+++ b/tools/report_analyzer.py
@@ -162,7 +162,9 @@ def main():
         "--table",
         dest="table_path",
         default="report_summary.md",
-        help="path to output md file with summary, defaults to \"report_summary.md\"")
+        help=
+        "path to output md file with summary, defaults to \"report_summary.md\""
+    )
     args = parser.parse_args()
 
     reportA = get_data(args.report_compare)

From 4b18f85649b8728ec9e4751128051ecc9d29a344 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Fri, 5 Feb 2021 20:35:02 +0100
Subject: [PATCH 45/69] perform full sv-tests

---
 .github/workflows/sv-tests-ci.yml | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index c85f8b37ae837..342022df10bcf 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -34,20 +34,20 @@ jobs:
       fail-fast: false
       matrix:
         env:
-          #- { JOB_NAME: iverilog, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: moore, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: odin_ii, MAKEFLAGS: -j2 }
+          - { JOB_NAME: iverilog, MAKEFLAGS: -j2 }
+          - { JOB_NAME: moore, MAKEFLAGS: -j2 }
+          - { JOB_NAME: odin_ii, MAKEFLAGS: -j2 }
           - { JOB_NAME: slang, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: sv-parser, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: surelog, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: tree-sitter-verilog, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: yosys, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: antmicro-yosys-complete, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: verible, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: verilator, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: uhdm-integration-verilator, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: uhdm-integration-yosys, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: zachjs-sv2v, MAKEFLAGS: -j2 }
+          - { JOB_NAME: sv-parser, MAKEFLAGS: -j2 }
+          - { JOB_NAME: surelog, MAKEFLAGS: -j2 }
+          - { JOB_NAME: tree-sitter-verilog, MAKEFLAGS: -j2 }
+          - { JOB_NAME: yosys, MAKEFLAGS: -j2 }
+          - { JOB_NAME: antmicro-yosys-complete, MAKEFLAGS: -j2 }
+          - { JOB_NAME: verible, MAKEFLAGS: -j2 }
+          - { JOB_NAME: verilator, MAKEFLAGS: -j2 }
+          - { JOB_NAME: uhdm-integration-verilator, MAKEFLAGS: -j2 }
+          - { JOB_NAME: uhdm-integration-yosys, MAKEFLAGS: -j2 }
+          - { JOB_NAME: zachjs-sv2v, MAKEFLAGS: -j2 }
 
     name: ${{ matrix.env.JOB_NAME }}
     env: ${{ matrix.env }}

From 725223230b406fae110b97476305608146618f3e Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Mon, 8 Feb 2021 13:37:59 +0100
Subject: [PATCH 46/69] Revert "perform full sv-tests"

This reverts commit 4b18f85649b8728ec9e4751128051ecc9d29a344.
---
 .github/workflows/sv-tests-ci.yml | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 342022df10bcf..c85f8b37ae837 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -34,20 +34,20 @@ jobs:
       fail-fast: false
       matrix:
         env:
-          - { JOB_NAME: iverilog, MAKEFLAGS: -j2 }
-          - { JOB_NAME: moore, MAKEFLAGS: -j2 }
-          - { JOB_NAME: odin_ii, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: iverilog, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: moore, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: odin_ii, MAKEFLAGS: -j2 }
           - { JOB_NAME: slang, MAKEFLAGS: -j2 }
-          - { JOB_NAME: sv-parser, MAKEFLAGS: -j2 }
-          - { JOB_NAME: surelog, MAKEFLAGS: -j2 }
-          - { JOB_NAME: tree-sitter-verilog, MAKEFLAGS: -j2 }
-          - { JOB_NAME: yosys, MAKEFLAGS: -j2 }
-          - { JOB_NAME: antmicro-yosys-complete, MAKEFLAGS: -j2 }
-          - { JOB_NAME: verible, MAKEFLAGS: -j2 }
-          - { JOB_NAME: verilator, MAKEFLAGS: -j2 }
-          - { JOB_NAME: uhdm-integration-verilator, MAKEFLAGS: -j2 }
-          - { JOB_NAME: uhdm-integration-yosys, MAKEFLAGS: -j2 }
-          - { JOB_NAME: zachjs-sv2v, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: sv-parser, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: surelog, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: tree-sitter-verilog, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: yosys, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: antmicro-yosys-complete, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: verible, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: verilator, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: uhdm-integration-verilator, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: uhdm-integration-yosys, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: zachjs-sv2v, MAKEFLAGS: -j2 }
 
     name: ${{ matrix.env.JOB_NAME }}
     env: ${{ matrix.env }}

From 07e5d7c9aaf9ee0b3872a43debf0dbff3cceda13 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Mon, 8 Feb 2021 13:43:51 +0100
Subject: [PATCH 47/69] check deleting previous summary commit

---
 .github/workflows/sv-tests-ci.yml | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index c85f8b37ae837..c51d9ec4cbe2e 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -119,8 +119,17 @@ jobs:
           path: ./out/report
       - name: Display structure of downloaded files
         run: ls -R ./out/
-      - uses: harupy/comment-on-pr@master
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - id: get-comment-body
+        run: |
+          body=$(cat ../../out/report/tests_summary.md)
+          body="${body//'%'/'%25'}"
+          body="${body//$'\n'/'%0A'}"
+          body="${body//$'\r'/'%0D'}"
+          echo ::set-output name=body::$body
+      - name: Post comment
+        uses: KeisukeYamashita/create-comment@v1
         with:
-          filename: ../../out/report/tests_summary.md
+          check-only-first-line: "true"
+          unique: "true"
+          token: ${{ secrets.GITHUB_TOKEN }}
+          comment: ${{ steps.get-comment-body.outputs.body }}

From 764c6feaf6b3e977ffd17443d6798d534caf91dc Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Mon, 8 Feb 2021 13:48:24 +0100
Subject: [PATCH 48/69] test comment editing

---
 .github/workflows/sv-tests-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index c51d9ec4cbe2e..ba714c0f86531 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -119,7 +119,7 @@ jobs:
           path: ./out/report
       - name: Display structure of downloaded files
         run: ls -R ./out/
-      - id: get-comment-body
+      - id: get-comment-bodyy
         run: |
           body=$(cat ../../out/report/tests_summary.md)
           body="${body//'%'/'%25'}"

From 8968487e324a273fbd13c99488a754d2080362f3 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Mon, 8 Feb 2021 14:20:22 +0100
Subject: [PATCH 49/69] fix relative path from previous gh-action

---
 .github/workflows/sv-tests-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index ba714c0f86531..45b64cc6bfc38 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -121,7 +121,7 @@ jobs:
         run: ls -R ./out/
       - id: get-comment-bodyy
         run: |
-          body=$(cat ../../out/report/tests_summary.md)
+          body=$(cat ./out/report/tests_summary.md)
           body="${body//'%'/'%25'}"
           body="${body//$'\n'/'%0A'}"
           body="${body//$'\r'/'%0D'}"

From 6c32047874e5fa70eb5442eb7aeae682b71c04ee Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Mon, 8 Feb 2021 14:25:46 +0100
Subject: [PATCH 50/69] testing coment delete and update

---
 .github/workflows/sv-tests-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 45b64cc6bfc38..675f4f369c763 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -119,7 +119,7 @@ jobs:
           path: ./out/report
       - name: Display structure of downloaded files
         run: ls -R ./out/
-      - id: get-comment-bodyy
+      - id: get-comment-body
         run: |
           body=$(cat ./out/report/tests_summary.md)
           body="${body//'%'/'%25'}"

From ef9f87057690b13e39be44604769a90f5f17c0a7 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Mon, 8 Feb 2021 15:02:50 +0100
Subject: [PATCH 51/69] test edit

---
 .github/workflows/sv-tests-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 675f4f369c763..0fbf401962aa1 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -126,7 +126,7 @@ jobs:
           body="${body//$'\n'/'%0A'}"
           body="${body//$'\r'/'%0D'}"
           echo ::set-output name=body::$body
-      - name: Post comment
+      - name: Post commentt
         uses: KeisukeYamashita/create-comment@v1
         with:
           check-only-first-line: "true"

From 75c3464f5dc12dda585613b40cccbcde3c65a237 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Mon, 8 Feb 2021 15:34:55 +0100
Subject: [PATCH 52/69] bring back normal step name

---
 .github/workflows/sv-tests-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 0fbf401962aa1..675f4f369c763 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -126,7 +126,7 @@ jobs:
           body="${body//$'\n'/'%0A'}"
           body="${body//$'\r'/'%0D'}"
           echo ::set-output name=body::$body
-      - name: Post commentt
+      - name: Post comment
         uses: KeisukeYamashita/create-comment@v1
         with:
           check-only-first-line: "true"

From 67e42ce67c4c0395ca4112d6d6b573f553a9963b Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Mon, 8 Feb 2021 15:35:17 +0100
Subject: [PATCH 53/69] Revert "Revert "perform full sv-tests""

This reverts commit 725223230b406fae110b97476305608146618f3e.
---
 .github/workflows/sv-tests-ci.yml | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 675f4f369c763..128873e057d00 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -34,20 +34,20 @@ jobs:
       fail-fast: false
       matrix:
         env:
-          #- { JOB_NAME: iverilog, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: moore, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: odin_ii, MAKEFLAGS: -j2 }
+          - { JOB_NAME: iverilog, MAKEFLAGS: -j2 }
+          - { JOB_NAME: moore, MAKEFLAGS: -j2 }
+          - { JOB_NAME: odin_ii, MAKEFLAGS: -j2 }
           - { JOB_NAME: slang, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: sv-parser, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: surelog, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: tree-sitter-verilog, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: yosys, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: antmicro-yosys-complete, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: verible, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: verilator, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: uhdm-integration-verilator, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: uhdm-integration-yosys, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: zachjs-sv2v, MAKEFLAGS: -j2 }
+          - { JOB_NAME: sv-parser, MAKEFLAGS: -j2 }
+          - { JOB_NAME: surelog, MAKEFLAGS: -j2 }
+          - { JOB_NAME: tree-sitter-verilog, MAKEFLAGS: -j2 }
+          - { JOB_NAME: yosys, MAKEFLAGS: -j2 }
+          - { JOB_NAME: antmicro-yosys-complete, MAKEFLAGS: -j2 }
+          - { JOB_NAME: verible, MAKEFLAGS: -j2 }
+          - { JOB_NAME: verilator, MAKEFLAGS: -j2 }
+          - { JOB_NAME: uhdm-integration-verilator, MAKEFLAGS: -j2 }
+          - { JOB_NAME: uhdm-integration-yosys, MAKEFLAGS: -j2 }
+          - { JOB_NAME: zachjs-sv2v, MAKEFLAGS: -j2 }
 
     name: ${{ matrix.env.JOB_NAME }}
     env: ${{ matrix.env }}

From 4655fc26b2cb8122c66848554b949d82f3ccd582 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Mon, 8 Feb 2021 16:30:10 +0100
Subject: [PATCH 54/69] stop doing tool-specific analysis

---
 .github/workflows/analyze.sh             | 25 ------------------------
 .github/workflows/summary.sh             |  6 +++---
 .github/workflows/sv-tests-ci.yml        |  7 +++----
 .github/workflows/tool_report_prepare.sh | 17 ++++++++++++++++
 4 files changed, 23 insertions(+), 32 deletions(-)
 delete mode 100755 .github/workflows/analyze.sh
 create mode 100755 .github/workflows/tool_report_prepare.sh

diff --git a/.github/workflows/analyze.sh b/.github/workflows/analyze.sh
deleted file mode 100755
index e1e28e3bead13..0000000000000
--- a/.github/workflows/analyze.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/bash
-
-ANALYZER=$PWD"/tools/report_analyzer.py"
-OUT_DIR=$PWD"/out/report/"
-COMPARE_REPORT=$OUT_DIR"/report.csv"
-BASE_REPORT=$OUT_DIR"/base_report.csv"
-CHANGES_SUMMARY=$OUT_DIR"/"$JOB_NAME"_changes_summary.json"
-
-set -x
-set -e
-
-source "$HOME/miniconda/etc/profile.d/conda.sh"
-hash -r
-conda activate sv-test-env
-hash -r
-
-# Get base report from sv-tests master run
-wget https://symbiflow.github.io/sv-tests-results/report.csv -O $BASE_REPORT
-
-python $ANALYZER $COMPARE_REPORT $BASE_REPORT -o $CHANGES_SUMMARY
-
-mv $COMPARE_REPORT $OUT_DIR"/"$JOB_NAME"_report.csv"
-
-set +e
-set +x
diff --git a/.github/workflows/summary.sh b/.github/workflows/summary.sh
index 61d3118b92254..478d793223470 100755
--- a/.github/workflows/summary.sh
+++ b/.github/workflows/summary.sh
@@ -26,15 +26,15 @@ conda info -a
 wget https://symbiflow.github.io/sv-tests-results/report.csv -O $BASE_REPORT
 
 # Delete headers from all report.csv
-for file in $(find ./out/changes_summary_* -name "*.csv" -print); do
+for file in $(find ./out/report_* -name "*.csv" -print); do
 	sed -i.backup 1,1d $file
 done
 
 # concatenate test reports
-cat $(find ./out/changes_summary_* -name "*.csv" -print) >> $COMPARE_REPORT
+cat $(find ./out/report_* -name "*.csv" -print) >> $COMPARE_REPORT
 
 # Insert header at the first line of concatenated report
-sed -i 1i\ $(cat $(find ./out/changes_summary_* -name "*.csv.backup" -print | head -1) | head -1) $COMPARE_REPORT
+sed -i 1i\ $(cat $(find ./out/report_* -name "*.csv.backup" -print | head -1) | head -1) $COMPARE_REPORT
 
 python $ANALYZER $COMPARE_REPORT $BASE_REPORT -o $CHANGES_SUMMARY_JSON -t $CHANGES_SUMMARY_MD
 
diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 128873e057d00..7540e5086a7b9 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -64,14 +64,13 @@ jobs:
       - name: Run
         run:
           ./.github/workflows/run.sh
-      - name: Analyze
+      - name: Prepare Report
         run:
-          ./.github/workflows/analyze.sh
+          ./.github/workflows/tool_report_prepare.sh
       - uses: actions/upload-artifact@v2
         with:
-          name: changes_summary_${{ matrix.env.JOB_NAME }}
+          name: report_${{ matrix.env.JOB_NAME }}
           path: |
-            ./out/report/${{ matrix.env.JOB_NAME }}_changes_summary.json
             ./out/report/${{ matrix.env.JOB_NAME }}_report.csv
 
   Summary:
diff --git a/.github/workflows/tool_report_prepare.sh b/.github/workflows/tool_report_prepare.sh
new file mode 100755
index 0000000000000..d799dd76eaba8
--- /dev/null
+++ b/.github/workflows/tool_report_prepare.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+OUT_DIR=$PWD"/out/report/"
+COMPARE_REPORT=$OUT_DIR"/report.csv"
+
+set -x
+set -e
+
+source "$HOME/miniconda/etc/profile.d/conda.sh"
+hash -r
+conda activate sv-test-env
+hash -r
+
+mv $COMPARE_REPORT $OUT_DIR"/"$JOB_NAME"_report.csv"
+
+set +e
+set +x

From 48cf8e9e46417a35409b7adf56b4e5738ee2c596 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Mon, 8 Feb 2021 16:31:05 +0100
Subject: [PATCH 55/69] test only one tool

---
 .github/workflows/sv-tests-ci.yml | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 7540e5086a7b9..ebe6e8b4fbdf0 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -34,20 +34,20 @@ jobs:
       fail-fast: false
       matrix:
         env:
-          - { JOB_NAME: iverilog, MAKEFLAGS: -j2 }
-          - { JOB_NAME: moore, MAKEFLAGS: -j2 }
-          - { JOB_NAME: odin_ii, MAKEFLAGS: -j2 }
-          - { JOB_NAME: slang, MAKEFLAGS: -j2 }
-          - { JOB_NAME: sv-parser, MAKEFLAGS: -j2 }
-          - { JOB_NAME: surelog, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: iverilog, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: moore, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: odin_ii, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: slang, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: sv-parser, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: surelog, MAKEFLAGS: -j2 }
           - { JOB_NAME: tree-sitter-verilog, MAKEFLAGS: -j2 }
-          - { JOB_NAME: yosys, MAKEFLAGS: -j2 }
-          - { JOB_NAME: antmicro-yosys-complete, MAKEFLAGS: -j2 }
-          - { JOB_NAME: verible, MAKEFLAGS: -j2 }
-          - { JOB_NAME: verilator, MAKEFLAGS: -j2 }
-          - { JOB_NAME: uhdm-integration-verilator, MAKEFLAGS: -j2 }
-          - { JOB_NAME: uhdm-integration-yosys, MAKEFLAGS: -j2 }
-          - { JOB_NAME: zachjs-sv2v, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: yosys, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: antmicro-yosys-complete, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: verible, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: verilator, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: uhdm-integration-verilator, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: uhdm-integration-yosys, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: zachjs-sv2v, MAKEFLAGS: -j2 }
 
     name: ${{ matrix.env.JOB_NAME }}
     env: ${{ matrix.env }}

From 79cf8a84adcd1d6d45f92464baba7387c6239b87 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Mon, 8 Feb 2021 16:58:15 +0100
Subject: [PATCH 56/69] delete old artifacts

---
 .github/workflows/sv-tests-ci.yml | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index ebe6e8b4fbdf0..f81b9c934d868 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -101,6 +101,19 @@ jobs:
             ./out/report/tests_summary.md
             ./out/report/tests_report.csv
             ./out/report/base_report.csv
+      - id: get-artifacts-to-delete
+        run: |
+          artifacts=$(ls ./out/report_*)
+          echo $artifacts
+          artifacts="${artifacts//'%'/'%25'}"
+          artifacts="${artifacts//$'\n'/'%0A'}"
+          artifacts="${artifacts//$'\r'/'%0D'}"
+          echo ::set-output name=artifacts::$artifacts
+          echo $artifacts
+      - name: Delete Old Artifacts
+        uses: geekyeggo/delete-artifact@v1
+          with:
+            name: ${{ steps.get-artifacts-to-delete.outputs.artifacts }}
 
   Comment:
     if: github.event_name == 'pull_request'

From dc11ed76bade0b2f537c79bd4b1e7c49a48e4e0a Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Mon, 8 Feb 2021 16:58:42 +0100
Subject: [PATCH 57/69] try yosys tests

---
 .github/workflows/sv-tests-ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index f81b9c934d868..39b64c8e48a59 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -40,8 +40,8 @@ jobs:
           #- { JOB_NAME: slang, MAKEFLAGS: -j2 }
           #- { JOB_NAME: sv-parser, MAKEFLAGS: -j2 }
           #- { JOB_NAME: surelog, MAKEFLAGS: -j2 }
-          - { JOB_NAME: tree-sitter-verilog, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: yosys, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: tree-sitter-verilog, MAKEFLAGS: -j2 }
+          - { JOB_NAME: yosys, MAKEFLAGS: -j2 }
           #- { JOB_NAME: antmicro-yosys-complete, MAKEFLAGS: -j2 }
           #- { JOB_NAME: verible, MAKEFLAGS: -j2 }
           #- { JOB_NAME: verilator, MAKEFLAGS: -j2 }

From f9d2f0f4211460f48d86e30ccfad9c531bb52718 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Mon, 8 Feb 2021 17:04:22 +0100
Subject: [PATCH 58/69] Revert "try yosys tests"

This reverts commit dc11ed76bade0b2f537c79bd4b1e7c49a48e4e0a.
---
 .github/workflows/sv-tests-ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 39b64c8e48a59..f81b9c934d868 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -40,8 +40,8 @@ jobs:
           #- { JOB_NAME: slang, MAKEFLAGS: -j2 }
           #- { JOB_NAME: sv-parser, MAKEFLAGS: -j2 }
           #- { JOB_NAME: surelog, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: tree-sitter-verilog, MAKEFLAGS: -j2 }
-          - { JOB_NAME: yosys, MAKEFLAGS: -j2 }
+          - { JOB_NAME: tree-sitter-verilog, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: yosys, MAKEFLAGS: -j2 }
           #- { JOB_NAME: antmicro-yosys-complete, MAKEFLAGS: -j2 }
           #- { JOB_NAME: verible, MAKEFLAGS: -j2 }
           #- { JOB_NAME: verilator, MAKEFLAGS: -j2 }

From e5fe5aa2b49b6212231cf3f925692367d1dacd19 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Mon, 8 Feb 2021 17:12:39 +0100
Subject: [PATCH 59/69] empty commit sync


From fef8b35e2e8b01cfc70c43bc0f7ba252c7266f87 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Mon, 8 Feb 2021 17:16:31 +0100
Subject: [PATCH 60/69] fix yaml syntax error

---
 .github/workflows/sv-tests-ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index f81b9c934d868..5809ad3ec37d5 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -112,8 +112,8 @@ jobs:
           echo $artifacts
       - name: Delete Old Artifacts
         uses: geekyeggo/delete-artifact@v1
-          with:
-            name: ${{ steps.get-artifacts-to-delete.outputs.artifacts }}
+        with:
+          name: ${{ steps.get-artifacts-to-delete.outputs.artifacts }}
 
   Comment:
     if: github.event_name == 'pull_request'

From a84e93b2a75719d5c0b5745d7ad79591fa3c4b57 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Mon, 8 Feb 2021 17:22:17 +0100
Subject: [PATCH 61/69] comment job cleanup

---
 .github/workflows/sv-tests-ci.yml | 4 ----
 .github/workflows/template.md     | 1 -
 2 files changed, 5 deletions(-)
 delete mode 100644 .github/workflows/template.md

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 5809ad3ec37d5..53bd30b851a4f 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -121,16 +121,12 @@ jobs:
     runs-on: ubuntu-18.04
     needs: Summary
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v2
       - name: Prepare output directories
         run: mkdir -p out/report
       - uses: actions/download-artifact@v2
         with:
           name: tests_summary
           path: ./out/report
-      - name: Display structure of downloaded files
-        run: ls -R ./out/
       - id: get-comment-body
         run: |
           body=$(cat ./out/report/tests_summary.md)
diff --git a/.github/workflows/template.md b/.github/workflows/template.md
deleted file mode 100644
index a042389697364..0000000000000
--- a/.github/workflows/template.md
+++ /dev/null
@@ -1 +0,0 @@
-hello world!

From 6d4617bb482b4a6a12e4c1091b7ff0d59ec5a4fe Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Mon, 8 Feb 2021 17:29:57 +0100
Subject: [PATCH 62/69] more cleanup

---
 .github/workflows/sv-tests-ci.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 53bd30b851a4f..a15d12887944f 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -88,8 +88,6 @@ jobs:
       - uses: actions/download-artifact@v2
         with:
           path: ./out/
-      - name: Display structure of downloaded files
-        run: ls -R ./out/
       - name: Summary
         run:
           ./.github/workflows/summary.sh

From 1b290588361ac2eef8a79f1297122c34dedd39fa Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Mon, 8 Feb 2021 17:30:29 +0100
Subject: [PATCH 63/69] test with 2 tools

---
 .github/workflows/sv-tests-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index a15d12887944f..551ad4a6f0724 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -37,7 +37,7 @@ jobs:
           #- { JOB_NAME: iverilog, MAKEFLAGS: -j2 }
           #- { JOB_NAME: moore, MAKEFLAGS: -j2 }
           #- { JOB_NAME: odin_ii, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: slang, MAKEFLAGS: -j2 }
+          - { JOB_NAME: slang, MAKEFLAGS: -j2 }
           #- { JOB_NAME: sv-parser, MAKEFLAGS: -j2 }
           #- { JOB_NAME: surelog, MAKEFLAGS: -j2 }
           - { JOB_NAME: tree-sitter-verilog, MAKEFLAGS: -j2 }

From 10f0e575ed918d485bb3c62bf6975d9452aaf16e Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Mon, 8 Feb 2021 17:41:19 +0100
Subject: [PATCH 64/69] update kokoro script

---
 .github/kokoro/run.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/kokoro/run.sh b/.github/kokoro/run.sh
index 449c1861bda73..ab47097de2f18 100755
--- a/.github/kokoro/run.sh
+++ b/.github/kokoro/run.sh
@@ -87,12 +87,13 @@ echo "----------------------------------------"
 	OUT_DIR=$GIT_CHECKOUT"/out/report/"
 	COMPARE_REPORT=$OUT_DIR"/report.csv"
 	BASE_REPORT=$OUT_DIR"/base_report.csv"
-	CHANGES_SUMMARY=$OUT_DIR"/changes_summary.json"
+	CHANGES_SUMMARY_JSON=$OUT_DIR"/tests_summary.json"
+	CHANGES_SUMMARY_MD=$OUT_DIR"/tests_summary.md"
 
 	# Get base report from sv-tests master run
 	wget https://symbiflow.github.io/sv-tests-results/report.csv -O $BASE_REPORT
 
-	python $ANALYZER $COMPARE_REPORT $BASE_REPORT -o $CHANGES_SUMMARY
+	python $ANALYZER $COMPARE_REPORT $BASE_REPORT -o $CHANGES_SUMMARY_JSON -t $CHANGES_SUMMARY_MD
 }
 echo "----------------------------------------"
 

From 413d43aa237e83f06f7c3077989fbb2218fdd90b Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Mon, 8 Feb 2021 17:41:55 +0100
Subject: [PATCH 65/69] do full test

---
 .github/workflows/sv-tests-ci.yml | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 551ad4a6f0724..51a942c8fa93e 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -34,20 +34,20 @@ jobs:
       fail-fast: false
       matrix:
         env:
-          #- { JOB_NAME: iverilog, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: moore, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: odin_ii, MAKEFLAGS: -j2 }
+          - { JOB_NAME: iverilog, MAKEFLAGS: -j2 }
+          - { JOB_NAME: moore, MAKEFLAGS: -j2 }
+          - { JOB_NAME: odin_ii, MAKEFLAGS: -j2 }
           - { JOB_NAME: slang, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: sv-parser, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: surelog, MAKEFLAGS: -j2 }
+          - { JOB_NAME: sv-parser, MAKEFLAGS: -j2 }
+          - { JOB_NAME: surelog, MAKEFLAGS: -j2 }
           - { JOB_NAME: tree-sitter-verilog, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: yosys, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: antmicro-yosys-complete, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: verible, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: verilator, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: uhdm-integration-verilator, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: uhdm-integration-yosys, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: zachjs-sv2v, MAKEFLAGS: -j2 }
+          - { JOB_NAME: yosys, MAKEFLAGS: -j2 }
+          - { JOB_NAME: antmicro-yosys-complete, MAKEFLAGS: -j2 }
+          - { JOB_NAME: verible, MAKEFLAGS: -j2 }
+          - { JOB_NAME: verilator, MAKEFLAGS: -j2 }
+          - { JOB_NAME: uhdm-integration-verilator, MAKEFLAGS: -j2 }
+          - { JOB_NAME: uhdm-integration-yosys, MAKEFLAGS: -j2 }
+          - { JOB_NAME: zachjs-sv2v, MAKEFLAGS: -j2 }
 
     name: ${{ matrix.env.JOB_NAME }}
     env: ${{ matrix.env }}

From 882b2265d774999fe15be9f355bd973453b75341 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Mon, 8 Feb 2021 18:22:53 +0100
Subject: [PATCH 66/69] fix artifacts name lookup

---
 .github/workflows/sv-tests-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 51a942c8fa93e..0a2e404d6a007 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -101,7 +101,7 @@ jobs:
             ./out/report/base_report.csv
       - id: get-artifacts-to-delete
         run: |
-          artifacts=$(ls ./out/report_*)
+          artifacts=$(find ./out -type d -name 'report_*')
           echo $artifacts
           artifacts="${artifacts//'%'/'%25'}"
           artifacts="${artifacts//$'\n'/'%0A'}"

From 2949ab133a2003474c17d6bdc9c8decf243f5680 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Mon, 8 Feb 2021 18:23:21 +0100
Subject: [PATCH 67/69] test one

---
 .github/workflows/sv-tests-ci.yml | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 0a2e404d6a007..4075e0fdee12f 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -34,20 +34,20 @@ jobs:
       fail-fast: false
       matrix:
         env:
-          - { JOB_NAME: iverilog, MAKEFLAGS: -j2 }
-          - { JOB_NAME: moore, MAKEFLAGS: -j2 }
-          - { JOB_NAME: odin_ii, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: iverilog, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: moore, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: odin_ii, MAKEFLAGS: -j2 }
           - { JOB_NAME: slang, MAKEFLAGS: -j2 }
-          - { JOB_NAME: sv-parser, MAKEFLAGS: -j2 }
-          - { JOB_NAME: surelog, MAKEFLAGS: -j2 }
-          - { JOB_NAME: tree-sitter-verilog, MAKEFLAGS: -j2 }
-          - { JOB_NAME: yosys, MAKEFLAGS: -j2 }
-          - { JOB_NAME: antmicro-yosys-complete, MAKEFLAGS: -j2 }
-          - { JOB_NAME: verible, MAKEFLAGS: -j2 }
-          - { JOB_NAME: verilator, MAKEFLAGS: -j2 }
-          - { JOB_NAME: uhdm-integration-verilator, MAKEFLAGS: -j2 }
-          - { JOB_NAME: uhdm-integration-yosys, MAKEFLAGS: -j2 }
-          - { JOB_NAME: zachjs-sv2v, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: sv-parser, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: surelog, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: tree-sitter-verilog, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: yosys, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: antmicro-yosys-complete, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: verible, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: verilator, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: uhdm-integration-verilator, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: uhdm-integration-yosys, MAKEFLAGS: -j2 }
+          #- { JOB_NAME: zachjs-sv2v, MAKEFLAGS: -j2 }
 
     name: ${{ matrix.env.JOB_NAME }}
     env: ${{ matrix.env }}

From 9f44ae323664405428cf04b0464ae5b0b12dcc26 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Mon, 8 Feb 2021 19:02:15 +0100
Subject: [PATCH 68/69] once again artifact name lookup fix

---
 .github/workflows/sv-tests-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 4075e0fdee12f..57c746a0949a1 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -101,7 +101,7 @@ jobs:
             ./out/report/base_report.csv
       - id: get-artifacts-to-delete
         run: |
-          artifacts=$(find ./out -type d -name 'report_*')
+          artifacts=$(find ./out -type d -name 'report_*' -exec basename {} \;)
           echo $artifacts
           artifacts="${artifacts//'%'/'%25'}"
           artifacts="${artifacts//$'\n'/'%0A'}"

From 0429382cbad14c56c05ee52f600db42736e8bae5 Mon Sep 17 00:00:00 2001
From: Pawel Czarnecki <pczarnecki@antmicro.com>
Date: Mon, 8 Feb 2021 19:28:00 +0100
Subject: [PATCH 69/69] full tests that should finally work

---
 .github/workflows/sv-tests-ci.yml | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/sv-tests-ci.yml b/.github/workflows/sv-tests-ci.yml
index 57c746a0949a1..a0b62306464b6 100644
--- a/.github/workflows/sv-tests-ci.yml
+++ b/.github/workflows/sv-tests-ci.yml
@@ -34,20 +34,20 @@ jobs:
       fail-fast: false
       matrix:
         env:
-          #- { JOB_NAME: iverilog, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: moore, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: odin_ii, MAKEFLAGS: -j2 }
+          - { JOB_NAME: iverilog, MAKEFLAGS: -j2 }
+          - { JOB_NAME: moore, MAKEFLAGS: -j2 }
+          - { JOB_NAME: odin_ii, MAKEFLAGS: -j2 }
           - { JOB_NAME: slang, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: sv-parser, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: surelog, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: tree-sitter-verilog, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: yosys, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: antmicro-yosys-complete, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: verible, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: verilator, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: uhdm-integration-verilator, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: uhdm-integration-yosys, MAKEFLAGS: -j2 }
-          #- { JOB_NAME: zachjs-sv2v, MAKEFLAGS: -j2 }
+          - { JOB_NAME: sv-parser, MAKEFLAGS: -j2 }
+          - { JOB_NAME: surelog, MAKEFLAGS: -j2 }
+          - { JOB_NAME: tree-sitter-verilog, MAKEFLAGS: -j2 }
+          - { JOB_NAME: yosys, MAKEFLAGS: -j2 }
+          - { JOB_NAME: antmicro-yosys-complete, MAKEFLAGS: -j2 }
+          - { JOB_NAME: verible, MAKEFLAGS: -j2 }
+          - { JOB_NAME: verilator, MAKEFLAGS: -j2 }
+          - { JOB_NAME: uhdm-integration-verilator, MAKEFLAGS: -j2 }
+          - { JOB_NAME: uhdm-integration-yosys, MAKEFLAGS: -j2 }
+          - { JOB_NAME: zachjs-sv2v, MAKEFLAGS: -j2 }
 
     name: ${{ matrix.env.JOB_NAME }}
     env: ${{ matrix.env }}