perf(linux): build configs in parallel (default)
Linux built the 3 configs sequentially (~3x a single config); Windows already parallelizes. Run them concurrently via ThreadPoolExecutor, splitting make -j across configs to avoid CPU oversubscription. Symlink setup serialized with a lock. --no-parallel restores sequential. Brings Linux wall time close to Windows. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -15,8 +15,14 @@ import os
|
|||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
|
import threading
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
# Symlink creation is shared across configs; serialize it so parallel
|
||||||
|
# per-config builds don't race recreating the bundle symlinks.
|
||||||
|
_SYMLINK_LOCK = threading.Lock()
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[2]
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
|
||||||
OVERLAY = ROOT / "tools" / "linux" / "overlay"
|
OVERLAY = ROOT / "tools" / "linux" / "overlay"
|
||||||
@@ -158,7 +164,9 @@ def _build_one(
|
|||||||
bundle_dir = layout["bundle_dir"]
|
bundle_dir = layout["bundle_dir"]
|
||||||
|
|
||||||
# Symlinks are not committed to git; recreate them on Linux if missing.
|
# Symlinks are not committed to git; recreate them on Linux if missing.
|
||||||
|
# Serialized: concurrent per-config builds would race here.
|
||||||
if zip_version in VERSION_LAYOUT:
|
if zip_version in VERSION_LAYOUT:
|
||||||
|
with _SYMLINK_LOCK:
|
||||||
_ensure_bundle_symlinks(zip_version, bundle_dir, vendor_7zip)
|
_ensure_bundle_symlinks(zip_version, bundle_dir, vendor_7zip)
|
||||||
|
|
||||||
cfg = CONFIGS[cfg_name]
|
cfg = CONFIGS[cfg_name]
|
||||||
@@ -313,6 +321,18 @@ def main() -> int:
|
|||||||
help="Copy built DLLs to <repo>/dist/<config> instead of "
|
help="Copy built DLLs to <repo>/dist/<config> instead of "
|
||||||
"<repo>/plugins/<config>",
|
"<repo>/plugins/<config>",
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--parallel",
|
||||||
|
action="store_true",
|
||||||
|
default=True,
|
||||||
|
help="Build configs concurrently (default: True)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--no-parallel",
|
||||||
|
action="store_false",
|
||||||
|
dest="parallel",
|
||||||
|
help="Build configs one at a time",
|
||||||
|
)
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
_require_tool("make")
|
_require_tool("make")
|
||||||
@@ -329,6 +349,35 @@ def main() -> int:
|
|||||||
|
|
||||||
wanted = list(CONFIGS.keys()) if "all" in args.configs else args.configs
|
wanted = list(CONFIGS.keys()) if "all" in args.configs else args.configs
|
||||||
|
|
||||||
|
if args.parallel and len(wanted) > 1:
|
||||||
|
# Split make jobs across configs so 3 concurrent makes don't
|
||||||
|
# oversubscribe the CPU (mirrors the Windows parallel build).
|
||||||
|
per_jobs = max(1, jobs // len(wanted))
|
||||||
|
print(
|
||||||
|
f"[linux] Building {len(wanted)} configs in parallel "
|
||||||
|
f"(make -j{per_jobs} each)"
|
||||||
|
)
|
||||||
|
codes: dict[str, int] = {}
|
||||||
|
with ThreadPoolExecutor(max_workers=len(wanted)) as ex:
|
||||||
|
futs = {
|
||||||
|
ex.submit(
|
||||||
|
_build_one,
|
||||||
|
args.zip_version,
|
||||||
|
c,
|
||||||
|
args.verbose,
|
||||||
|
per_jobs,
|
||||||
|
args.clean,
|
||||||
|
args.cleanup_artifacts,
|
||||||
|
args.dist,
|
||||||
|
): c
|
||||||
|
for c in wanted
|
||||||
|
}
|
||||||
|
for fut, name in futs.items():
|
||||||
|
codes[name] = fut.result()
|
||||||
|
for name in wanted:
|
||||||
|
if codes[name] != 0:
|
||||||
|
return codes[name]
|
||||||
|
else:
|
||||||
for cfg_name in wanted:
|
for cfg_name in wanted:
|
||||||
code = _build_one(
|
code = _build_one(
|
||||||
args.zip_version,
|
args.zip_version,
|
||||||
|
|||||||
Reference in New Issue
Block a user