diff options
Diffstat (limited to 'nss/automation/taskcluster/graph/src/extend.js')
| -rw-r--r-- | nss/automation/taskcluster/graph/src/extend.js | 439 |
1 files changed, 322 insertions, 117 deletions
diff --git a/nss/automation/taskcluster/graph/src/extend.js b/nss/automation/taskcluster/graph/src/extend.js index a6a8fcb..d9f6938 100644 --- a/nss/automation/taskcluster/graph/src/extend.js +++ b/nss/automation/taskcluster/graph/src/extend.js @@ -5,7 +5,20 @@ import merge from "./merge"; import * as queue from "./queue"; -const LINUX_IMAGE = {name: "linux", path: "automation/taskcluster/docker"}; +const LINUX_IMAGE = { + name: "linux", + path: "automation/taskcluster/docker" +}; + +const LINUX_CLANG39_IMAGE = { + name: "linux-clang-3.9", + path: "automation/taskcluster/docker-clang-3.9" +}; + +const FUZZ_IMAGE = { + name: "fuzz", + path: "automation/taskcluster/docker-fuzz" +}; const WINDOWS_CHECKOUT_CMD = "bash -c \"hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss || " + @@ -17,33 +30,45 @@ const WINDOWS_CHECKOUT_CMD = queue.filter(task => { if (task.group == "Builds") { // Remove extra builds on {A,UB}San and ARM. - if (task.collection == "asan" || task.collection == "arm-debug" || - task.collection == "gyp-asan") { + if (task.collection == "asan" || task.platform == "aarch64") { return false; } - // Remove extra builds w/o libpkix for non-linux64-debug. - if (task.symbol == "noLibpkix" && - (task.platform != "linux64" || task.collection != "debug")) { + // Make modular builds only on Linux make. + if (task.symbol == "modular" && task.collection != "make") { return false; } } - if (task.tests == "bogo") { - // No BoGo tests on Windows. - if (task.platform == "windows2012-64") { + if (task.tests == "bogo" || task.tests == "interop") { + // No windows + if (task.platform == "windows2012-64" || + task.platform == "windows2012-32") { return false; } - // No BoGo tests on ARM. - if (task.collection == "arm-debug") { + // No ARM; TODO: enable + if (task.platform == "aarch64") { return false; } } - // GYP builds with -Ddisable_libpkix=1 by default. - if ((task.collection == "gyp" || task.collection == "gyp-asan") && - task.tests == "chains") { + // Only old make builds have -Ddisable_libpkix=0 and can run chain tests. + if (task.tests == "chains" && task.collection != "make") { + return false; + } + + if (task.group == "Test") { + // Don't run test builds on old make platforms + if (task.collection == "make") { + return false; + } + } + + // Don't run additional hardware tests on ARM (we don't have anything there). + if (task.group == "Cipher" && task.platform == "aarch64" && task.env && + (task.env.NSS_DISABLE_PCLMUL == "1" || task.env.NSS_DISABLE_HW_AES == "1" + || task.env.NSS_DISABLE_AVX == "1")) { return false; } @@ -51,29 +76,18 @@ queue.filter(task => { }); queue.map(task => { - if (task.collection == "asan" || task.collection == "gyp-asan") { + if (task.collection == "asan") { // CRMF and FIPS tests still leak, unfortunately. if (task.tests == "crmf" || task.tests == "fips") { task.env.ASAN_OPTIONS = "detect_leaks=0"; } } - if (task.collection == "arm-debug") { - // These tests take quite some time on our poor ARM devices. - if (task.tests == "chains" || (task.tests == "ssl" && task.cycle == "standard")) { - task.maxRunTime = 14400; - } - } - // Windows is slow. if (task.platform == "windows2012-64" && task.tests == "chains") { task.maxRunTime = 7200; } - // Enable TLS 1.3 for every task. - task.env = task.env || {}; - task.env.NSS_ENABLE_TLS_1_3 = "1"; - return task; }); @@ -81,58 +95,48 @@ queue.map(task => { export default async function main() { await scheduleLinux("Linux 32 (opt)", { - env: {BUILD_OPT: "1"}, platform: "linux32", image: LINUX_IMAGE - }); + }, "-m32 --opt"); await scheduleLinux("Linux 32 (debug)", { platform: "linux32", collection: "debug", image: LINUX_IMAGE - }); + }, "-m32"); await scheduleLinux("Linux 64 (opt)", { - env: {USE_64: "1", BUILD_OPT: "1"}, platform: "linux64", image: LINUX_IMAGE - }); + }, "--opt"); await scheduleLinux("Linux 64 (debug)", { - env: {USE_64: "1"}, platform: "linux64", collection: "debug", image: LINUX_IMAGE }); - await scheduleLinux("Linux 64 (debug, gyp)", { + await scheduleLinux("Linux 64 (debug, make)", { + env: {USE_64: "1"}, + platform: "linux64", + image: LINUX_IMAGE, + collection: "make", command: [ - "/bin/bash", - "-c", - "bin/checkout.sh && nss/automation/taskcluster/scripts/build_gyp.sh" + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/build.sh" ], - platform: "linux64", - collection: "gyp", - image: LINUX_IMAGE }); - await scheduleLinux("Linux 64 (debug, gyp, asan, ubsan)", { + await scheduleLinux("Linux 32 (debug, make)", { + platform: "linux32", + image: LINUX_IMAGE, + collection: "make", command: [ - "/bin/bash", - "-c", - "bin/checkout.sh && nss/automation/taskcluster/scripts/build_gyp.sh -g -v --ubsan --asan" + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/build.sh" ], - env: { - ASAN_OPTIONS: "detect_odr_violation=0", // bug 1316276 - UBSAN_OPTIONS: "print_stacktrace=1", - NSS_DISABLE_ARENA_FREE_LIST: "1", - NSS_DISABLE_UNLOAD: "1", - CC: "clang", - CCC: "clang++" - }, - platform: "linux64", - collection: "gyp-asan", - image: LINUX_IMAGE }); await scheduleLinux("Linux 64 (ASan, debug)", { @@ -142,49 +146,87 @@ export default async function main() { NSS_DISABLE_UNLOAD: "1", CC: "clang", CCC: "clang++", - USE_UBSAN: "1", - USE_ASAN: "1", - USE_64: "1" }, platform: "linux64", collection: "asan", - image: LINUX_IMAGE - }); + image: LINUX_IMAGE, + features: ["allowPtrace"], + }, "--ubsan --asan"); + + await scheduleWindows("Windows 2012 64 (debug, make)", { + platform: "windows2012-64", + collection: "make", + env: {USE_64: "1"} + }, "build.sh"); + + await scheduleWindows("Windows 2012 32 (debug, make)", { + platform: "windows2012-32", + collection: "make" + }, "build.sh"); await scheduleWindows("Windows 2012 64 (opt)", { - env: {BUILD_OPT: "1"} - }); + platform: "windows2012-64", + }, "build_gyp.sh --opt"); await scheduleWindows("Windows 2012 64 (debug)", { + platform: "windows2012-64", collection: "debug" - }); + }, "build_gyp.sh"); - await scheduleFuzzing(); + await scheduleWindows("Windows 2012 32 (opt)", { + platform: "windows2012-32", + }, "build_gyp.sh --opt -m32"); - await scheduleTestBuilds(); + await scheduleWindows("Windows 2012 32 (debug)", { + platform: "windows2012-32", + collection: "debug" + }, "build_gyp.sh -m32"); + + await scheduleFuzzing(); + await scheduleFuzzing32(); await scheduleTools(); - await scheduleLinux("Linux 32 (ARM, debug)", { - image: "franziskus/nss-arm-ci", + let aarch64_base = { + image: "franziskus/nss-aarch64-ci", provisioner: "localprovisioner", - collection: "arm-debug", - workerType: "nss-rpi", - platform: "linux32", - maxRunTime: 7200, - tier: 3 - }); + workerType: "nss-aarch64", + platform: "aarch64", + maxRunTime: 7200 + }; + + await scheduleLinux("Linux AArch64 (debug)", + merge({ + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/build_gyp.sh" + ], + collection: "debug", + }, aarch64_base) + ); + + await scheduleLinux("Linux AArch64 (opt)", + merge({ + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/build_gyp.sh --opt" + ], + collection: "opt", + }, aarch64_base) + ); } /*****************************************************************************/ -async function scheduleLinux(name, base) { +async function scheduleLinux(name, base, args = "") { // Build base definition. let build_base = merge({ command: [ "/bin/bash", "-c", - "bin/checkout.sh && nss/automation/taskcluster/scripts/build.sh" + "bin/checkout.sh && nss/automation/taskcluster/scripts/build_gyp.sh " + args ], artifacts: { public: { @@ -224,12 +266,12 @@ async function scheduleLinux(name, base) { // Extra builds. let extra_base = merge({group: "Builds"}, build_base); queue.scheduleTask(merge(extra_base, { - name: `${name} w/ clang-3.9`, + name: `${name} w/ clang-4.0`, env: { CC: "clang", CCC: "clang++", }, - symbol: "clang-3.9" + symbol: "clang-4.0" })); queue.scheduleTask(merge(extra_base, { @@ -251,30 +293,54 @@ async function scheduleLinux(name, base) { })); queue.scheduleTask(merge(extra_base, { - name: `${name} w/ NSS_DISABLE_LIBPKIX=1`, - env: {NSS_DISABLE_LIBPKIX: "1"}, - symbol: "noLibpkix" + name: `${name} w/ modular builds`, + env: {NSS_BUILD_MODULAR: "1"}, + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/build.sh", + ], + symbol: "modular" })); + await scheduleTestBuilds(merge(base, {group: "Test"}), args); + return queue.submit(); } /*****************************************************************************/ +function scheduleFuzzingRun(base, name, target, max_len, symbol = null, corpus = null) { + const MAX_FUZZ_TIME = 300; + + queue.scheduleTask(merge(base, { + name, + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/fuzz.sh " + + `${target} nss/fuzz/corpus/${corpus || target} ` + + `-max_total_time=${MAX_FUZZ_TIME} ` + + `-max_len=${max_len}` + ], + symbol: symbol || name + })); +} + async function scheduleFuzzing() { let base = { env: { - // bug 1316276 - ASAN_OPTIONS: "allocator_may_return_null=1:detect_odr_violation=0", + ASAN_OPTIONS: "allocator_may_return_null=1:detect_stack_use_after_return=1", UBSAN_OPTIONS: "print_stacktrace=1", NSS_DISABLE_ARENA_FREE_LIST: "1", NSS_DISABLE_UNLOAD: "1", CC: "clang", CCC: "clang++" }, + features: ["allowPtrace"], platform: "linux64", collection: "fuzz", - image: LINUX_IMAGE + image: FUZZ_IMAGE }; // Build base definition. @@ -301,9 +367,22 @@ async function scheduleFuzzing() { name: "Linux x64 (debug, fuzz)" })); + // The task that builds NSPR+NSS (TLS fuzzing mode). + let task_build_tls = queue.scheduleTask(merge(build_base, { + name: "Linux x64 (debug, TLS fuzz)", + symbol: "B", + group: "TLS", + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && " + + "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz=tls" + ], + })); + // Schedule tests. queue.scheduleTask(merge(base, { - parent: task_build, + parent: task_build_tls, name: "Gtests", command: [ "/bin/bash", @@ -317,56 +396,155 @@ async function scheduleFuzzing() { kind: "test" })); - queue.scheduleTask(merge(base, { - parent: task_build, - name: "Cert", + // Schedule fuzzing runs. + let run_base = merge(base, {parent: task_build, kind: "test"}); + scheduleFuzzingRun(run_base, "CertDN", "certDN", 4096); + scheduleFuzzingRun(run_base, "QuickDER", "quickder", 10000); + + // Schedule MPI fuzzing runs. + let mpi_base = merge(run_base, {group: "MPI"}); + let mpi_names = ["add", "addmod", "div", "expmod", "mod", "mulmod", "sqr", + "sqrmod", "sub", "submod"]; + for (let name of mpi_names) { + scheduleFuzzingRun(mpi_base, `MPI (${name})`, `mpi-${name}`, 4096, name); + } + scheduleFuzzingRun(mpi_base, `MPI (invmod)`, `mpi-invmod`, 256, "invmod"); + + // Schedule TLS fuzzing runs (non-fuzzing mode). + let tls_base = merge(run_base, {group: "TLS"}); + scheduleFuzzingRun(tls_base, "TLS Client", "tls-client", 20000, "client-nfm", + "tls-client-no_fuzzer_mode"); + scheduleFuzzingRun(tls_base, "TLS Server", "tls-server", 20000, "server-nfm", + "tls-server-no_fuzzer_mode"); + scheduleFuzzingRun(tls_base, "DTLS Client", "dtls-client", 20000, + "dtls-client-nfm", "dtls-client-no_fuzzer_mode"); + scheduleFuzzingRun(tls_base, "DTLS Server", "dtls-server", 20000, + "dtls-server-nfm", "dtls-server-no_fuzzer_mode"); + + // Schedule TLS fuzzing runs (fuzzing mode). + let tls_fm_base = merge(tls_base, {parent: task_build_tls}); + scheduleFuzzingRun(tls_fm_base, "TLS Client", "tls-client", 20000, "client"); + scheduleFuzzingRun(tls_fm_base, "TLS Server", "tls-server", 20000, "server"); + scheduleFuzzingRun(tls_fm_base, "DTLS Client", "dtls-client", 20000, "dtls-client"); + scheduleFuzzingRun(tls_fm_base, "DTLS Server", "dtls-server", 20000, "dtls-server"); + + return queue.submit(); +} + +async function scheduleFuzzing32() { + let base = { + env: { + ASAN_OPTIONS: "allocator_may_return_null=1:detect_stack_use_after_return=1", + UBSAN_OPTIONS: "print_stacktrace=1", + NSS_DISABLE_ARENA_FREE_LIST: "1", + NSS_DISABLE_UNLOAD: "1", + CC: "clang", + CCC: "clang++" + }, + features: ["allowPtrace"], + platform: "linux32", + collection: "fuzz", + image: FUZZ_IMAGE + }; + + // Build base definition. + let build_base = merge({ command: [ "/bin/bash", "-c", - "bin/checkout.sh && nss/automation/taskcluster/scripts/fuzz.sh " + - "cert nss/fuzz/corpus/cert -max_total_time=300" + "bin/checkout.sh && " + + "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz -m32" ], - // Need a privileged docker container to remove this. - env: {ASAN_OPTIONS: "detect_leaks=0"}, - symbol: "SCert", - kind: "test" + artifacts: { + public: { + expires: 24 * 7, + type: "directory", + path: "/home/worker/artifacts" + } + }, + kind: "build", + symbol: "B" + }, base); + + // The task that builds NSPR+NSS. + let task_build = queue.scheduleTask(merge(build_base, { + name: "Linux 32 (debug, fuzz)" })); + // The task that builds NSPR+NSS (TLS fuzzing mode). + let task_build_tls = queue.scheduleTask(merge(build_base, { + name: "Linux 32 (debug, TLS fuzz)", + symbol: "B", + group: "TLS", + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && " + + "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz=tls -m32" + ], + })); + + // Schedule tests. queue.scheduleTask(merge(base, { - parent: task_build, - name: "SPKI", + parent: task_build_tls, + name: "Gtests", command: [ "/bin/bash", "-c", - "bin/checkout.sh && nss/automation/taskcluster/scripts/fuzz.sh " + - "spki nss/fuzz/corpus/spki -max_total_time=300" + "bin/checkout.sh && nss/automation/taskcluster/scripts/run_tests.sh" ], - // Need a privileged docker container to remove this. - env: {ASAN_OPTIONS: "detect_leaks=0"}, - symbol: "SPKI", + env: {GTESTFILTER: "*Fuzz*"}, + tests: "ssl_gtests gtests", + cycle: "standard", + symbol: "Gtest", kind: "test" })); + // Schedule fuzzing runs. + let run_base = merge(base, {parent: task_build, kind: "test"}); + scheduleFuzzingRun(run_base, "CertDN", "certDN", 4096); + scheduleFuzzingRun(run_base, "QuickDER", "quickder", 10000); + + // Schedule MPI fuzzing runs. + let mpi_base = merge(run_base, {group: "MPI"}); + let mpi_names = ["add", "addmod", "div", "expmod", "mod", "mulmod", "sqr", + "sqrmod", "sub", "submod"]; + for (let name of mpi_names) { + scheduleFuzzingRun(mpi_base, `MPI (${name})`, `mpi-${name}`, 4096, name); + } + scheduleFuzzingRun(mpi_base, `MPI (invmod)`, `mpi-invmod`, 256, "invmod"); + + // Schedule TLS fuzzing runs (non-fuzzing mode). + let tls_base = merge(run_base, {group: "TLS"}); + scheduleFuzzingRun(tls_base, "TLS Client", "tls-client", 20000, "client-nfm", + "tls-client-no_fuzzer_mode"); + scheduleFuzzingRun(tls_base, "TLS Server", "tls-server", 20000, "server-nfm", + "tls-server-no_fuzzer_mode"); + scheduleFuzzingRun(tls_base, "DTLS Client", "dtls-client", 20000, + "dtls-client-nfm", "dtls-client-no_fuzzer_mode"); + scheduleFuzzingRun(tls_base, "DTLS Server", "dtls-server", 20000, + "dtls-server-nfm", "dtls-server-no_fuzzer_mode"); + + // Schedule TLS fuzzing runs (fuzzing mode). + let tls_fm_base = merge(tls_base, {parent: task_build_tls}); + scheduleFuzzingRun(tls_fm_base, "TLS Client", "tls-client", 20000, "client"); + scheduleFuzzingRun(tls_fm_base, "TLS Server", "tls-server", 20000, "server"); + scheduleFuzzingRun(tls_fm_base, "DTLS Client", "dtls-client", 20000, "dtls-client"); + scheduleFuzzingRun(tls_fm_base, "DTLS Server", "dtls-server", 20000, "dtls-server"); + return queue.submit(); } /*****************************************************************************/ -async function scheduleTestBuilds() { - let base = { - platform: "linux64", - collection: "gyp", - group: "Test", - image: LINUX_IMAGE - }; - +async function scheduleTestBuilds(base, args = "") { // Build base definition. let build = merge({ command: [ "/bin/bash", "-c", "bin/checkout.sh && " + - "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --test" + "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --test --ct-verif " + args ], artifacts: { public: { @@ -377,7 +555,7 @@ async function scheduleTestBuilds() { }, kind: "build", symbol: "B", - name: "Linux 64 (debug, gyp, test)" + name: "Linux 64 (debug, test)" }, base); // The task that builds NSPR+NSS. @@ -397,6 +575,19 @@ async function scheduleTestBuilds() { symbol: "mpi", kind: "test" })); + queue.scheduleTask(merge(base, { + parent: task_build, + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/run_tests.sh" + ], + name: "Gtests", + symbol: "Gtest", + tests: "gtests", + cycle: "standard", + kind: "test" + })); return queue.submit(); } @@ -404,10 +595,9 @@ async function scheduleTestBuilds() { /*****************************************************************************/ -async function scheduleWindows(name, base) { +async function scheduleWindows(name, base, build_script) { base = merge(base, { workerType: "nss-win2012r2", - platform: "windows2012-64", env: { PATH: "c:\\mozilla-build\\python;c:\\mozilla-build\\msys\\local\\bin;" + "c:\\mozilla-build\\7zip;c:\\mozilla-build\\info-zip;" + @@ -417,7 +607,6 @@ async function scheduleWindows(name, base) { "c:\\mozilla-build\\wget", DOMSUF: "localdomain", HOST: "localhost", - USE_64: "1" } }); @@ -425,7 +614,7 @@ async function scheduleWindows(name, base) { let build_base = merge(base, { command: [ WINDOWS_CHECKOUT_CMD, - "bash -c nss/automation/taskcluster/windows/build.sh" + `bash -c 'nss/automation/taskcluster/windows/${build_script}'` ], artifacts: [{ expires: 24 * 7, @@ -475,10 +664,25 @@ function scheduleTests(task_build, task_cert, test_base) { name: "Bogo tests", symbol: "Bogo", tests: "bogo", cycle: "standard" })); queue.scheduleTask(merge(no_cert_base, { + name: "Interop tests", symbol: "Interop", tests: "interop", cycle: "standard" + })); + queue.scheduleTask(merge(no_cert_base, { name: "Chains tests", symbol: "Chains", tests: "chains" })); queue.scheduleTask(merge(no_cert_base, { - name: "Cipher tests", symbol: "Cipher", tests: "cipher" + name: "Cipher tests", symbol: "Default", tests: "cipher", group: "Cipher" + })); + queue.scheduleTask(merge(no_cert_base, { + name: "Cipher tests", symbol: "NoAESNI", tests: "cipher", + env: {NSS_DISABLE_HW_AES: "1"}, group: "Cipher" + })); + queue.scheduleTask(merge(no_cert_base, { + name: "Cipher tests", symbol: "NoPCLMUL", tests: "cipher", + env: {NSS_DISABLE_PCLMUL: "1"}, group: "Cipher" + })); + queue.scheduleTask(merge(no_cert_base, { + name: "Cipher tests", symbol: "NoAVX", tests: "cipher", + env: {NSS_DISABLE_AVX: "1"}, group: "Cipher" })); queue.scheduleTask(merge(no_cert_base, { name: "EC tests", symbol: "EC", tests: "ec" @@ -531,7 +735,6 @@ function scheduleTests(task_build, task_cert, test_base) { async function scheduleTools() { let base = { - image: LINUX_IMAGE, platform: "nss-tools", kind: "test" }; @@ -539,6 +742,7 @@ async function scheduleTools() { queue.scheduleTask(merge(base, { symbol: "clang-format-3.9", name: "clang-format-3.9", + image: LINUX_CLANG39_IMAGE, command: [ "/bin/bash", "-c", @@ -547,8 +751,9 @@ async function scheduleTools() { })); queue.scheduleTask(merge(base, { - symbol: "scan-build-3.9", - name: "scan-build-3.9", + symbol: "scan-build-4.0", + name: "scan-build-4.0", + image: LINUX_IMAGE, env: { USE_64: "1", CC: "clang", |