From 7b8a65ea8f53f6ecc4883bc70859f8342c9442fb Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Thu, 25 Jan 2024 14:43:52 -0800 Subject: [PATCH 1/4] Slim down the LLVM builds Disable lots of tools and features that we don't need for Halide buildbots. On my Mac, this drops the install dir from ~6GB to ~2GB, and the compressed size from ~1.5GB to ~0.5GB. --- master/master.cfg | 167 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 154 insertions(+), 13 deletions(-) diff --git a/master/master.cfg b/master/master.cfg index 0e2e70c..f3462e3 100644 --- a/master/master.cfg +++ b/master/master.cfg @@ -721,6 +721,158 @@ def get_cmake_build_command(builder_type, build_dir, targets=None): return cmd +# options in LLVM that we always want OFF because they aren't needed +# for Halide buildbot usage. +_LLVM_OFF_OPTS = [ + "CLANG_ENABLE_ARCMT" + "CLANG_ENABLE_CLANGD" + "CLANG_ENABLE_STATIC_ANALYZER" + "CLANG_INCLUDE_DOCS" + "CLANG_INSTALL_SCANBUILD" + "CLANG_INSTALL_SCANVIEW" + "CLANG_PLUGIN_SUPPORT" + "CLANG_TIDY_ENABLE_STATIC_ANALYZER" + "CLANG_TOOL_APINOTES_TEST_BUILD" + "CLANG_TOOL_ARCMT_TEST_BUILD" + "CLANG_TOOL_C_ARCMT_TEST_BUILD" + "CLANG_TOOL_C_INDEX_TEST_BUILD" + "CLANG_TOOL_CLANG_CHECK_BUILD" + "CLANG_TOOL_CLANG_DIFF_BUILD" + "CLANG_TOOL_CLANG_EXTDEF_MAPPING_BUILD" + "CLANG_TOOL_CLANG_FORMAT_BUILD" + "CLANG_TOOL_CLANG_FORMAT_VS_BUILD" + "CLANG_TOOL_CLANG_FUZZER_BUILD" + "CLANG_TOOL_CLANG_IMPORT_TEST_BUILD" + "CLANG_TOOL_CLANG_LINKER_WRAPPER_BUILD" + "CLANG_TOOL_CLANG_OFFLOAD_BUNDLER_BUILD" + "CLANG_TOOL_CLANG_OFFLOAD_PACKAGER_BUILD" + "CLANG_TOOL_CLANG_REFACTOR_BUILD" + "CLANG_TOOL_CLANG_RENAME_BUILD" + "CLANG_TOOL_CLANG_REPL_BUILD" + "CLANG_TOOL_CLANG_SCAN_DEPS_BUILD" + "CLANG_TOOL_CLANG_SHLIB_BUILD" + "CLANG_TOOL_DIAGTOOL_BUILD" + "CLANG_TOOL_DICTIONARY_BUILD" + "CLANG_TOOL_LIBCLANG_BUILD" + "CLANG_TOOL_NVPTX_ARCH_BUILD" + "CLANG_TOOL_SCAN_BUILD_BUILD" + "CLANG_TOOL_SCAN_BUILD_PY_BUILD" + "CLANG_TOOL_SCAN_VIEW_BUILD" + "CLANG_TOOLS_EXTRA_INCLUDE_DOCS" + "LLVM_BUILD_32_BITS" + "LLVM_BUILD_UTILS" + "LLVM_ENABLE_BINDINGS" + "LLVM_ENABLE_CURL" + "LLVM_ENABLE_DIA_SDK" + "LLVM_ENABLE_HTTPLIB" + "LLVM_ENABLE_IDE" + "LLVM_ENABLE_LIBXML2" + "LLVM_ENABLE_OCAMLDOC" + "LLVM_ENABLE_PLUGINS" + "LLVM_ENABLE_TERMINFO" + "LLVM_ENABLE_WARNINGS" + "LLVM_ENABLE_ZSTD", + "LLVM_INCLUDE_BENCHMARKS" + "LLVM_INCLUDE_DOCS" + "LLVM_INCLUDE_EXAMPLES" + "LLVM_INCLUDE_TESTS" + "LLVM_INCLUDE_UTILS" + "LLVM_TOOL_BUGPOINT_BUILD" + "LLVM_TOOL_BUGPOINT_PASSES_BUILD" + "LLVM_TOOL_DSYMUTIL_BUILD" + "LLVM_TOOL_DXIL_DIS_BUILD" + "LLVM_TOOL_GOLD_BUILD" + "LLVM_TOOL_LLC_BUILD" + "LLVM_TOOL_LLI_BUILD" + "LLVM_TOOL_LLVM_AR_BUILD" + # "LLVM_TOOL_LLVM_AS_BUILD" -- no: ClangConfig.cmake needs this + # "LLVM_TOOL_LLVM_AS_FUZZER_BUILD" -- no: ClangConfig.cmake needs this + "LLVM_TOOL_LLVM_BCANALYZER_BUILD" + "LLVM_TOOL_LLVM_C_TEST_BUILD" + "LLVM_TOOL_LLVM_CAT_BUILD" + "LLVM_TOOL_LLVM_CFI_VERIFY_BUILD" + "LLVM_TOOL_LLVM_COV_BUILD" + "LLVM_TOOL_LLVM_CVTRES_BUILD" + "LLVM_TOOL_LLVM_CXXDUMP_BUILD" + "LLVM_TOOL_LLVM_CXXFILT_BUILD" + "LLVM_TOOL_LLVM_CXXMAP_BUILD" + "LLVM_TOOL_LLVM_DEBUGINFO_ANALYZER_BUILD" + "LLVM_TOOL_LLVM_DEBUGINFOD_BUILD" + "LLVM_TOOL_LLVM_DEBUGINFOD_FIND_BUILD" + "LLVM_TOOL_LLVM_DIFF_BUILD" + "LLVM_TOOL_LLVM_DIS_BUILD" + "LLVM_TOOL_LLVM_DIS_FUZZER_BUILD" + "LLVM_TOOL_LLVM_DLANG_DEMANGLE_FUZZER_BUILD" + "LLVM_TOOL_LLVM_DWARFDUMP_BUILD" + "LLVM_TOOL_LLVM_DWARFUTIL_BUILD" + "LLVM_TOOL_LLVM_DWP_BUILD" + "LLVM_TOOL_LLVM_EXEGESIS_BUILD" + "LLVM_TOOL_LLVM_EXTRACT_BUILD" + "LLVM_TOOL_LLVM_GSYMUTIL_BUILD" + "LLVM_TOOL_LLVM_IFS_BUILD" + "LLVM_TOOL_LLVM_ISEL_FUZZER_BUILD" + "LLVM_TOOL_LLVM_ITANIUM_DEMANGLE_FUZZER_BUILD" + "LLVM_TOOL_LLVM_JITLINK_BUILD" + "LLVM_TOOL_LLVM_JITLISTENER_BUILD" + "LLVM_TOOL_LLVM_LIBTOOL_DARWIN_BUILD" + "LLVM_TOOL_LLVM_LINK_BUILD" + "LLVM_TOOL_LLVM_LIPO_BUILD" + "LLVM_TOOL_LLVM_LTO2_BUILD" + "LLVM_TOOL_LLVM_LTO_BUILD" + "LLVM_TOOL_LLVM_MC_ASSEMBLE_FUZZER_BUILD" + "LLVM_TOOL_LLVM_MC_BUILD" + "LLVM_TOOL_LLVM_MC_DISASSEMBLE_FUZZER_BUILD" + "LLVM_TOOL_LLVM_MCA_BUILD" + "LLVM_TOOL_LLVM_MICROSOFT_DEMANGLE_FUZZER_BUILD" + "LLVM_TOOL_LLVM_ML_BUILD" + "LLVM_TOOL_LLVM_MODEXTRACT_BUILD" + "LLVM_TOOL_LLVM_MT_BUILD" + "LLVM_TOOL_LLVM_NM_BUILD" + "LLVM_TOOL_LLVM_OBJCOPY_BUILD" + "LLVM_TOOL_LLVM_OBJDUMP_BUILD" + "LLVM_TOOL_LLVM_OPT_FUZZER_BUILD" + "LLVM_TOOL_LLVM_OPT_REPORT_BUILD" + "LLVM_TOOL_LLVM_PDBUTIL_BUILD" + "LLVM_TOOL_LLVM_PROFDATA_BUILD" + "LLVM_TOOL_LLVM_PROFGEN_BUILD" + "LLVM_TOOL_LLVM_RC_BUILD" + "LLVM_TOOL_LLVM_READOBJ_BUILD" + "LLVM_TOOL_LLVM_READTAPI_BUILD" + "LLVM_TOOL_LLVM_REDUCE_BUILD" + "LLVM_TOOL_LLVM_REMARKUTIL_BUILD" + "LLVM_TOOL_LLVM_RTDYLD_BUILD" + "LLVM_TOOL_LLVM_RUST_DEMANGLE_FUZZER_BUILD" + "LLVM_TOOL_LLVM_SHLIB_BUILD" + "LLVM_TOOL_LLVM_SIM_BUILD" + "LLVM_TOOL_LLVM_SIZE_BUILD" + "LLVM_TOOL_LLVM_SPECIAL_CASE_LIST_FUZZER_BUILD" + "LLVM_TOOL_LLVM_SPLIT_BUILD" + "LLVM_TOOL_LLVM_STRESS_BUILD" + "LLVM_TOOL_LLVM_STRINGS_BUILD" + "LLVM_TOOL_LLVM_SYMBOLIZER_BUILD" + "LLVM_TOOL_LLVM_TLI_CHECKER_BUILD" + "LLVM_TOOL_LLVM_UNDNAME_BUILD" + "LLVM_TOOL_LLVM_XRAY_BUILD" + "LLVM_TOOL_LLVM_YAML_NUMERIC_PARSER_FUZZER_BUILD" + "LLVM_TOOL_LLVM_YAML_PARSER_FUZZER_BUILD" + "LLVM_TOOL_LTO_BUILD" + "LLVM_TOOL_MLIR_BUILD" + "LLVM_TOOL_OBJ2YAML_BUILD" + "LLVM_TOOL_OPENMP_BUILD" + "LLVM_TOOL_OPT_BUILD" + "LLVM_TOOL_OPT_VIEWER_BUILD" + "LLVM_TOOL_POLLY_BUILD" + "LLVM_TOOL_PSTL_BUILD" + "LLVM_TOOL_REMARKS_SHLIB_BUILD" + "LLVM_TOOL_SANCOV_BUILD" + "LLVM_TOOL_SANSTATS_BUILD" + "LLVM_TOOL_SPIRV_TOOLS_BUILD" + "LLVM_TOOL_VERIFY_USELISTORDER_BUILD" + "LLVM_TOOL_VFABI_DEMANGLE_FUZZER_BUILD" + "LLVM_TOOL_XCODE_TOOLCHAIN_BUILD" + "LLVM_TOOL_YAML2OBJ_BUILD" +] + def get_llvm_cmake_definitions(builder_type): # Keep sorted! @@ -729,23 +881,12 @@ def get_llvm_cmake_definitions(builder_type): 'CMAKE_INSTALL_PREFIX': get_llvm_install_path(builder_type), 'LLVM_BUILD_32_BITS': ('ON' if builder_type.bits == 32 else 'OFF'), 'LLVM_ENABLE_ASSERTIONS': 'ON', - 'LLVM_ENABLE_BINDINGS': 'OFF', - 'LLVM_ENABLE_CURL': 'OFF', - 'LLVM_ENABLE_DIA_SDK': 'OFF', - 'LLVM_ENABLE_HTTPLIB': 'OFF', - 'LLVM_ENABLE_IDE': 'OFF', - 'LLVM_ENABLE_LIBXML2': 'OFF', - 'LLVM_ENABLE_OCAMLDOC': 'OFF', 'LLVM_ENABLE_RTTI': 'ON', - 'LLVM_ENABLE_TERMINFO': 'OFF', - 'LLVM_ENABLE_WARNINGS': 'OFF', # silence them, it's not like we're gonna fix them 'LLVM_ENABLE_ZLIB': 'ON', - 'LLVM_ENABLE_ZSTD': 'OFF', - 'LLVM_INCLUDE_BENCHMARKS': 'OFF', - 'LLVM_INCLUDE_EXAMPLES': 'OFF', - 'LLVM_INCLUDE_TESTS': 'OFF', 'LLVM_TARGETS_TO_BUILD': 'X86;ARM;NVPTX;AArch64;Hexagon;PowerPC;WebAssembly', } + for off in _LLVM_OFF_OPTS: + definitions[off]='OFF' if builder_type.bits == 32: definitions['CMAKE_FIND_ROOT_PATH_MODE_INCLUDE'] = "ONLY" From b7b9caed97683cdd82f1e002ecac7fb0f5b7241d Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Thu, 25 Jan 2024 14:52:01 -0800 Subject: [PATCH 2/4] Update master.cfg --- master/master.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/master/master.cfg b/master/master.cfg index f3462e3..03af369 100644 --- a/master/master.cfg +++ b/master/master.cfg @@ -721,6 +721,7 @@ def get_cmake_build_command(builder_type, build_dir, targets=None): return cmd + # options in LLVM that we always want OFF because they aren't needed # for Halide buildbot usage. _LLVM_OFF_OPTS = [ @@ -885,6 +886,7 @@ def get_llvm_cmake_definitions(builder_type): 'LLVM_ENABLE_ZLIB': 'ON', 'LLVM_TARGETS_TO_BUILD': 'X86;ARM;NVPTX;AArch64;Hexagon;PowerPC;WebAssembly', } + for off in _LLVM_OFF_OPTS: definitions[off]='OFF' From ccac70f4e802b76e140f391778663fc122f99639 Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Thu, 25 Jan 2024 14:53:34 -0800 Subject: [PATCH 3/4] Update master.cfg --- master/master.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/master/master.cfg b/master/master.cfg index 03af369..89f77c6 100644 --- a/master/master.cfg +++ b/master/master.cfg @@ -888,7 +888,7 @@ def get_llvm_cmake_definitions(builder_type): } for off in _LLVM_OFF_OPTS: - definitions[off]='OFF' + definitions[off] = 'OFF' if builder_type.bits == 32: definitions['CMAKE_FIND_ROOT_PATH_MODE_INCLUDE'] = "ONLY" From 80a62bb41136f5863427360fef1a167ee24517b6 Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Thu, 25 Jan 2024 15:06:22 -0800 Subject: [PATCH 4/4] Update master.cfg --- master/master.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/master/master.cfg b/master/master.cfg index 89f77c6..b89ac01 100644 --- a/master/master.cfg +++ b/master/master.cfg @@ -760,7 +760,6 @@ _LLVM_OFF_OPTS = [ "CLANG_TOOL_SCAN_BUILD_PY_BUILD" "CLANG_TOOL_SCAN_VIEW_BUILD" "CLANG_TOOLS_EXTRA_INCLUDE_DOCS" - "LLVM_BUILD_32_BITS" "LLVM_BUILD_UTILS" "LLVM_ENABLE_BINDINGS" "LLVM_ENABLE_CURL"