Skip to content

ci: Include integration tests as part of coverage. #752

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
eyal0 merged 6 commits into from
Jan 15, 2026
Merged

ci: Include integration tests as part of coverage. #752

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
259e58c
ci: Enable code coverage for geos 3.13.1.
eyal0 Jan 14, 2026
80f465a
ci: Use more CPUs in Windows build and runs.
eyal0 Jan 14, 2026
8f6ec81
test: Better support for Windows in integration_tests.py.
eyal0 Jan 14, 2026
ddbb6bc
ci: Add support for running integration tests on more platforms.
eyal0 Jan 14, 2026
229e392
ci: Don't use valgrind with code coverage.
eyal0 Jan 15, 2026
97cf605
ci: Only use code coverage on g++-10.
eyal0 Jan 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 15 additions & 9 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,9 @@ jobs:
boost: '1_66'
include:
- os: ubuntu
boost: latest
compiler: g++
geos: latest
geos: '3.13.1'
code_coverage: "--enable-code-coverage"
compiler: 'g++-10' # Only code coverage with g++-10 because latest g++ coverage is slow.
- os: ubuntu
shell: '/usr/bin/bash -l -e -o pipefail {0}'
local_install_path: '$HOME/.local'
Expand Down Expand Up @@ -90,7 +89,7 @@ jobs:
- name: Windows specific setup
if: matrix.os == 'windows'
run: |
echo "export NUM_CPUS='$((`nproc --all`))'" >> ~/.bash_profile
echo "export NUM_CPUS='$((`nproc --all` * 4))'" >> ~/.bash_profile
- name: Ubuntu specific setup
if: matrix.os == 'ubuntu'
run: |
Expand Down Expand Up @@ -337,24 +336,31 @@ jobs:
continue-on-error: true
run: lcov --directory . -z
- name: Run unit tests
if: matrix.os != 'ubuntu'
if: ! (matrix.os == 'ubuntu' && ! matrix.code_coverage)
env:
VERBOSE: 1
SKIP_GERBERIMPORTER_TESTS_PNG: 1
run: make -j ${NUM_CPUS} check || (cat test-suite.log && false)
- name: Run unit tests with valgrind
if: matrix.os == 'ubuntu'
if: matrix.os == 'ubuntu' && ! matrix.code_coverage
env:
VERBOSE: 1
SKIP_GERBERIMPORTER_TESTS_PNG: 1
run: make -j ${NUM_CPUS} check-valgrind || (cat mem-test-suite.log && false)

- name: Run integration tests
if: matrix.geos != 'none' && matrix.geos != 'latest' && matrix.os == 'ubuntu'
if: matrix.geos != 'none' && matrix.geos != 'latest' && matrix.os != 'windows'
run: |
sudo apt-get install python3-setuptools
pip3 install --user wheel colour_runner unittest2 termcolor concurrencytest in_place
./integration_tests.py -j ${NUM_CPUS}
/usr/bin/python3 ./integration_tests.py --pcb2gcode-binary "$(pwd)/pcb2gcode" -j ${NUM_CPUS}
# Skip the Windows integration test because there are tiny discrepencies from expected outputs.
#- name: Run integration tests on Windows
# if: matrix.geos != 'none' && matrix.geos != 'latest' && matrix.os == 'windows'
# run: |
# pacman -S --noconfirm mingw-w64-x86_64-python-pip
# pip3 install --user wheel colour_runner unittest2 termcolor concurrencytest in_place
# python3 ./integration_tests.py --pcb2gcode-binary "$(pwd)/pcb2gcode.exe" -j ${NUM_CPUS}
- name: Gather coverage
if: matrix.code_coverage
run: |
Expand Down Expand Up @@ -482,7 +488,7 @@ jobs:
if: matrix.code_coverage
uses: actions/upload-artifact@v4
with:
name: version
name: version-${{ steps.sanitize-key.outputs.key }}.txt
path: version.txt
- name: Save cache
uses: actions/cache/save@v5
Expand Down
41 changes: 27 additions & 14 deletions integration_tests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,20 @@
import sys
import tempfile
import xml.etree.ElementTree

import colour_runner.runner
try:
import colour_runner.runner
colour_runner_available = True
except:
colour_runner_available = False
import in_place
import termcolor
import unittest

from concurrencytest import ConcurrentTestSuite, fork_for_tests
try:
from concurrencytest import ConcurrentTestSuite, fork_for_tests
concurrencytest_available = True
except:
concurrencytest_available = False

TestCase = collections.namedtuple("TestCase", ["name", "input_path", "args", "exit_code"])

Expand Down Expand Up @@ -213,19 +220,19 @@ def bigger(matchobj):
else:
svg_file.write(line)

def pcb2gcode_one_directory(self, input_path, cwd, args=None, exit_code=0):
def pcb2gcode_one_directory(self, input_path, pcb2gcode_binary, args=None, exit_code=0):
"""Run pcb2gcode once in one directory.

Current working directory remains unchanged at the end.

input_path: Where to run pcb2gcode
Returns the path to the output files created.
"""
pcb2gcode = os.path.join(cwd, "pcb2gcode")
cwd = os.getcwd() # Save this for later restoring the current working directory
actual_output_path = tempfile.mkdtemp()
os.chdir(input_path)
try:
cmd = [pcb2gcode]
cmd = [pcb2gcode_binary]
if not any("output-dir" in x for x in args):
cmd += ["--output-dir", actual_output_path]
cmd += args or []
Expand Down Expand Up @@ -298,7 +305,7 @@ def compare_directories(self, left, right, left_prefix="", right_prefix=""):
all_diffs += difflib.unified_diff(data0, data1, '"' + os.path.join(left_prefix, f) + '"', '"' + os.path.join(right_prefix, f) + '"')
return ''.join(all_diffs)

def run_one_directory(self, input_path, cwd, expected_output_path, test_prefix, args=[], exit_code=0):
def run_one_directory(self, input_path, pcb2gcode_binary, expected_output_path, test_prefix, args=[], exit_code=0):
"""Run pcb2gcode on a directory and return the diff as a string.

Returns an empty string if there is no mismatch.
Expand All @@ -307,7 +314,7 @@ def run_one_directory(self, input_path, cwd, expected_output_path, test_prefix,
expected_output_path: Path to expected outputs
test_prefix: Strin to prepend to all filenamess
"""
actual_output_path = self.pcb2gcode_one_directory(input_path, cwd, args, exit_code)
actual_output_path = self.pcb2gcode_one_directory(input_path, pcb2gcode_binary, args, exit_code)
if exit_code:
return ""
diff_text = self.compare_directories(expected_output_path, actual_output_path,
Expand All @@ -316,12 +323,13 @@ def run_one_directory(self, input_path, cwd, expected_output_path, test_prefix,
shutil.rmtree(actual_output_path)
return diff_text

def do_test_one(self, test_case, cwd):
def do_test_one(self, test_case, pcb2gcode_binary):
test_prefix = os.path.join(test_case.input_path, "expected")
cwd = os.path.dirname(pcb2gcode_binary)
input_path = os.path.join(cwd, test_case.input_path)
expected_output_path = os.path.join(cwd, test_case.input_path, "expected")
print(colored("\nRunning test case:\n" + "\n".join(" %s=%s" % (k,v) for k,v in test_case._asdict().items()), attrs=["bold"]), file=sys.stderr)
diff_text = self.run_one_directory(input_path, cwd, expected_output_path, test_prefix, test_case.args, test_case.exit_code)
diff_text = self.run_one_directory(input_path, pcb2gcode_binary, expected_output_path, test_prefix, test_case.args, test_case.exit_code)
self.assertFalse(bool(diff_text), 'Files don\'t match\n' + diff_text)

def cmp(x,y):
Expand All @@ -340,13 +348,15 @@ def cmp(x,y):
help='number of threads for running tests concurrently')
parser.add_argument('--tests', type=str, default="",
help='regex of tests to run')
parser.add_argument('--pcb2gcode-binary', type=str, default="",
help='path to pcb2gcode binary to run. The tests are expected to be in subdirectories of the directory containing the binary.')
args = parser.parse_args()
if args.tests:
TEST_CASES = [t for t in TEST_CASES if re.search(args.tests, t.name)]
cwd = os.getcwd()
pcb2gcode_binary = os.path.join(os.getcwd(), "pcb2gcode") if not args.pcb2gcode_binary else args.pcb2gcode_binary
def add_test_case(t):
def test_method(self):
self.do_test_one(t, cwd)
self.do_test_one(t, pcb2gcode_binary)
setattr(IntegrationTests, 'test_' + t.name, test_method)
test_method.__name__ = 'test_' + t.name
test_method.__doc__ = str(test_case)
Expand Down Expand Up @@ -384,8 +394,11 @@ def test_method(self):
test_loader.sortTestMethodsUsing = lambda x,y: cmp(all_test_names.index(x), all_test_names.index(y))
suite = test_loader.loadTestsFromTestCase(IntegrationTests)
if args.jobs > 1:
suite = ConcurrentTestSuite(suite, fork_for_tests(args.jobs))
if hasattr(sys.stderr, "isatty") and sys.stderr.isatty():
if concurrencytest_available:
suite = ConcurrentTestSuite(suite, fork_for_tests(args.jobs))
else:
print("WARNING: Module 'concurrencytest' not available. Running tests sequentially.", file=sys.stderr)
if colour_runner_available and hasattr(sys.stderr, "isatty") and sys.stderr.isatty():
test_result = colour_runner.runner.ColourTextTestRunner(verbosity=2).run(suite)
else:
test_result = unittest.TextTestRunner(verbosity=2).run(suite)
Expand Down