diff options
Diffstat (limited to 'nss/automation')
54 files changed, 1331 insertions, 257 deletions
diff --git a/nss/automation/abi-check/expected-report-libfreebl3.so.txt b/nss/automation/abi-check/expected-report-libfreebl3.so.txt new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/nss/automation/abi-check/expected-report-libfreebl3.so.txt diff --git a/nss/automation/abi-check/expected-report-libfreeblpriv3.so.txt b/nss/automation/abi-check/expected-report-libfreeblpriv3.so.txt new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/nss/automation/abi-check/expected-report-libfreeblpriv3.so.txt diff --git a/nss/automation/abi-check/expected-report-libnspr4.so.txt b/nss/automation/abi-check/expected-report-libnspr4.so.txt new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/nss/automation/abi-check/expected-report-libnspr4.so.txt diff --git a/nss/automation/abi-check/expected-report-libnss3.so.txt b/nss/automation/abi-check/expected-report-libnss3.so.txt new file mode 100644 index 0000000..1bdbb40 --- /dev/null +++ b/nss/automation/abi-check/expected-report-libnss3.so.txt @@ -0,0 +1,12 @@ +Functions changes summary: 0 Removed, 0 Changed, 6 Added functions +Variables changes summary: 0 Removed, 0 Changed, 0 Added variable + +6 Added functions: + + 'function SECStatus CERT_GetCertIsPerm(const CERTCertificate*, PRBool*)' {CERT_GetCertIsPerm@@NSS_3.31} + 'function SECStatus CERT_GetCertIsTemp(const CERTCertificate*, PRBool*)' {CERT_GetCertIsTemp@@NSS_3.31} + 'function CERTCertificate* PK11_FindCertFromURI(const char*, void*)' {PK11_FindCertFromURI@@NSS_3.31} + 'function CERTCertList* PK11_FindCertsFromURI(const char*, void*)' {PK11_FindCertsFromURI@@NSS_3.31} + 'function char* PK11_GetModuleURI(SECMODModule*)' {PK11_GetModuleURI@@NSS_3.31} + 'function char* PK11_GetTokenURI()' {PK11_GetTokenURI@@NSS_3.31} + diff --git a/nss/automation/abi-check/expected-report-libnssckbi.so.txt b/nss/automation/abi-check/expected-report-libnssckbi.so.txt new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/nss/automation/abi-check/expected-report-libnssckbi.so.txt diff --git a/nss/automation/abi-check/expected-report-libnssdbm3.so.txt b/nss/automation/abi-check/expected-report-libnssdbm3.so.txt new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/nss/automation/abi-check/expected-report-libnssdbm3.so.txt diff --git a/nss/automation/abi-check/expected-report-libnsssysinit.so.txt b/nss/automation/abi-check/expected-report-libnsssysinit.so.txt new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/nss/automation/abi-check/expected-report-libnsssysinit.so.txt diff --git a/nss/automation/abi-check/expected-report-libnssutil3.so.txt b/nss/automation/abi-check/expected-report-libnssutil3.so.txt new file mode 100644 index 0000000..5c6e5d9 --- /dev/null +++ b/nss/automation/abi-check/expected-report-libnssutil3.so.txt @@ -0,0 +1,12 @@ +Functions changes summary: 0 Removed, 0 Changed, 6 Added functions +Variables changes summary: 0 Removed, 0 Changed, 0 Added variable + +6 Added functions: + + 'function void PK11URI_CreateURI(size_t, size_t)' {PK11URI_CreateURI@@NSSUTIL_3.31} + 'function void PK11URI_DestroyURI()' {PK11URI_DestroyURI@@NSSUTIL_3.31} + 'function char* PK11URI_FormatURI()' {PK11URI_FormatURI@@NSSUTIL_3.31} + 'function const char* PK11URI_GetPathAttribute(const char*)' {PK11URI_GetPathAttribute@@NSSUTIL_3.31} + 'function const char* PK11URI_GetQueryAttribute(const char*)' {PK11URI_GetQueryAttribute@@NSSUTIL_3.31} + 'function void PK11URI_ParseURI(const char*)' {PK11URI_ParseURI@@NSSUTIL_3.31} + diff --git a/nss/automation/abi-check/expected-report-libplc4.so.txt b/nss/automation/abi-check/expected-report-libplc4.so.txt new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/nss/automation/abi-check/expected-report-libplc4.so.txt diff --git a/nss/automation/abi-check/expected-report-libplds4.so.txt b/nss/automation/abi-check/expected-report-libplds4.so.txt new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/nss/automation/abi-check/expected-report-libplds4.so.txt diff --git a/nss/automation/abi-check/expected-report-libsmime3.so.txt b/nss/automation/abi-check/expected-report-libsmime3.so.txt new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/nss/automation/abi-check/expected-report-libsmime3.so.txt diff --git a/nss/automation/abi-check/expected-report-libsoftokn3.so.txt b/nss/automation/abi-check/expected-report-libsoftokn3.so.txt new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/nss/automation/abi-check/expected-report-libsoftokn3.so.txt diff --git a/nss/automation/abi-check/expected-report-libssl3.so.txt b/nss/automation/abi-check/expected-report-libssl3.so.txt new file mode 100644 index 0000000..b0a279d --- /dev/null +++ b/nss/automation/abi-check/expected-report-libssl3.so.txt @@ -0,0 +1,14 @@ +Functions changes summary: 0 Removed, 1 Changed, 0 Added function +Variables changes summary: 0 Removed, 0 Changed, 0 Added variable + +1 function with some indirect sub-type change: + + [C]'function SECStatus SSL_GetPreliminaryChannelInfo(SSLPreliminaryChannelInfo*, PRUintn)' at sslinfo.c:115:1 has some indirect sub-type changes: + parameter 1 of type 'SSLPreliminaryChannelInfo*' has sub-type changes: + in pointed to type 'typedef SSLPreliminaryChannelInfo' at sslt.h:318:1: + underlying type 'struct SSLPreliminaryChannelInfoStr' at sslt.h:287:1 changed: + type size changed from 128 to 160 bits + 1 data member insertion: + 'PRUint32 SSLPreliminaryChannelInfoStr::maxEarlyDataSize', at offset 128 (in bits) at sslt.h:314:1 + + diff --git a/nss/automation/abi-check/previous-nss-release b/nss/automation/abi-check/previous-nss-release new file mode 100644 index 0000000..1445e74 --- /dev/null +++ b/nss/automation/abi-check/previous-nss-release @@ -0,0 +1 @@ +NSS_3_30_BRANCH diff --git a/nss/automation/buildbot-slave/build.sh b/nss/automation/buildbot-slave/build.sh index 0917cec..3fc9148 100755 --- a/nss/automation/buildbot-slave/build.sh +++ b/nss/automation/buildbot-slave/build.sh @@ -19,6 +19,9 @@ proc_args() "--test-nss") TEST_NSS=1 ;; + "--check-abi") + CHECK_ABI=1 + ;; "--build-jss") BUILD_JSS=1 ;; @@ -40,6 +43,7 @@ proc_args() echo " --build-jss" echo " --test-nss" echo " --test-jss" + echo " --check-abi" exit 1 ;; esac @@ -215,6 +219,65 @@ test_nss() return ${RET} } +check_abi() +{ + print_log "######## NSS ABI CHECK - ${BITS} bits - ${OPT} ########" + print_log "######## creating temporary HG clones ########" + + rm -rf ${HGDIR}/baseline + mkdir ${HGDIR}/baseline + BASE_NSS=`cat ${HGDIR}/nss/automation/abi-check/previous-nss-release` + hg clone -u "${BASE_NSS}" "${HGDIR}/nss" "${HGDIR}/baseline/nss" + if [ $? -ne 0 ]; then + echo "invalid tag in automation/abi-check/previous-nss-release" + return 1 + fi + + BASE_NSPR=NSPR_$(head -1 ${HGDIR}/baseline/nss/automation/release/nspr-version.txt | cut -d . -f 1-2 | tr . _)_BRANCH + hg clone -u "${BASE_NSPR}" "${HGDIR}/nspr" "${HGDIR}/baseline/nspr" + if [ $? -ne 0 ]; then + echo "invalid tag ${BASE_NSPR} derived from ${BASE_NSS} automation/release/nspr-version.txt" + return 1 + fi + + print_log "######## building older NSPR/NSS ########" + pushd ${HGDIR}/baseline/nss + + print_log "$ ${MAKE} ${NSS_BUILD_TARGET}" + ${MAKE} ${NSS_BUILD_TARGET} 2>&1 | tee -a ${LOG_ALL} + RET=$? + print_result "NSS - build - ${BITS} bits - ${OPT}" ${RET} 0 + if [ ${RET} -ne 0 ]; then + tail -100 ${LOG_ALL} + return ${RET} + fi + popd + + ABI_REPORT=${OUTPUTDIR}/abi-diff.txt + rm -f ${ABI_REPORT} + PREVDIST=${HGDIR}/baseline/dist + NEWDIST=${HGDIR}/dist + ALL_SOs="libfreebl3.so libfreeblpriv3.so libnspr4.so libnss3.so libnssckbi.so libnssdbm3.so libnsssysinit.so libnssutil3.so libplc4.so libplds4.so libsmime3.so libsoftokn3.so libssl3.so" + for SO in ${ALL_SOs}; do + if [ ! -f nss/automation/abi-check/expected-report-$SO.txt ]; then + touch nss/automation/abi-check/expected-report-$SO.txt + fi + abidiff --hd1 $PREVDIST/public/ --hd2 $NEWDIST/public \ + $PREVDIST/*/lib/$SO $NEWDIST/*/lib/$SO \ + > nss/automation/abi-check/new-report-$SO.txt + diff -u nss/automation/abi-check/expected-report-$SO.txt \ + nss/automation/abi-check/new-report-$SO.txt >> ${ABI_REPORT} + done + + if [ -s ${ABI_REPORT} ]; then + print_log "FAILED: there are new unexpected ABI changes" + cat ${ABI_REPORT} + return 1 + fi + + return 0 +} + test_jss() { print_log "######## JSS - tests - ${BITS} bits - ${OPT} ########" @@ -243,6 +306,39 @@ test_jss() return ${RET} } +create_objdir_dist_link() +{ + # compute relevant 'dist' OBJDIR_NAME subdirectory names for JSS and NSS + OS_TARGET=`uname -s` + OS_RELEASE=`uname -r | sed 's/-.*//' | sed 's/-.*//' | cut -d . -f1,2` + CPU_TAG=_`uname -m` + # OBJDIR_NAME_COMPILER appears to be defined for NSS but not JSS + OBJDIR_NAME_COMPILER=_cc + LIBC_TAG=_glibc + IMPL_STRATEGY=_PTH + if [ "${RUN_BITS}" = "64" ]; then + OBJDIR_TAG=_${RUN_BITS}_${RUN_OPT}.OBJ + else + OBJDIR_TAG=_${RUN_OPT}.OBJ + fi + + # define NSS_OBJDIR_NAME + NSS_OBJDIR_NAME=${OS_TARGET}${OS_RELEASE}${CPU_TAG}${OBJDIR_NAME_COMPILER} + NSS_OBJDIR_NAME=${NSS_OBJDIR_NAME}${LIBC_TAG}${IMPL_STRATEGY}${OBJDIR_TAG} + print_log "create_objdir_dist_link(): NSS_OBJDIR_NAME='${NSS_OBJDIR_NAME}'" + + # define JSS_OBJDIR_NAME + JSS_OBJDIR_NAME=${OS_TARGET}${OS_RELEASE}${CPU_TAG} + JSS_OBJDIR_NAME=${JSS_OBJDIR_NAME}${LIBC_TAG}${IMPL_STRATEGY}${OBJDIR_TAG} + print_log "create_objdir_dist_link(): JSS_OBJDIR_NAME='${JSS_OBJDIR_NAME}'" + + if [ -e "${HGDIR}/dist/${NSS_OBJDIR_NAME}" ]; then + SOURCE=${HGDIR}/dist/${NSS_OBJDIR_NAME} + TARGET=${HGDIR}/dist/${JSS_OBJDIR_NAME} + ln -s ${SOURCE} ${TARGET} >/dev/null 2>&1 + fi +} + build_and_test() { if [ -n "${BUILD_NSS}" ]; then @@ -255,7 +351,13 @@ build_and_test() [ $? -eq 0 ] || return 1 fi + if [ -n "${CHECK_ABI}" ]; then + check_abi + [ $? -eq 0 ] || return 1 + fi + if [ -n "${BUILD_JSS}" ]; then + create_objdir_dist_link build_jss [ $? -eq 0 ] || return 1 fi @@ -326,6 +428,7 @@ main() { VALID=0 RET=1 + FAIL=0 for BITS in 32 64; do echo ${RUN_BITS} | grep ${BITS} > /dev/null @@ -338,7 +441,10 @@ main() set_env run_all RET=$? - print_log "### result of run_all is ${RET}" + print_log "### result of run_all is ${RET}" + if [ ${RET} -ne 0 ]; then + FAIL=${RET} + fi done done @@ -347,7 +453,7 @@ main() return 1 fi - return ${RET} + return ${FAIL} } #function killallsub() @@ -375,6 +481,8 @@ echo "tinderbox args: $0 $@" proc_args "$@" main -#RET=$? +RET=$? +print_log "### result of main is ${RET}" + #rm $IS_RUNNING_FILE -#exit ${RET} +exit ${RET} diff --git a/nss/automation/ossfuzz/build.sh b/nss/automation/ossfuzz/build.sh new file mode 100755 index 0000000..e967ea8 --- /dev/null +++ b/nss/automation/ossfuzz/build.sh @@ -0,0 +1,57 @@ +#!/bin/bash -eu +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +################################################################################ + +# List of targets disabled for oss-fuzz. +declare -A disabled=([pkcs8]=1) + +# List of targets we want to fuzz in TLS and non-TLS mode. +declare -A tls_targets=([tls-client]=1 [tls-server]=1 [dtls-client]=1 [dtls-server]=1) + +# Helper function that copies a fuzzer binary and its seed corpus. +copy_fuzzer() +{ + local fuzzer=$1 + local name=$2 + + # Copy the binary. + cp ../dist/Debug/bin/$fuzzer $OUT/$name + + # Zip and copy the corpus, if any. + if [ -d "$SRC/nss-corpus/$name" ]; then + zip $OUT/${name}_seed_corpus.zip $SRC/nss-corpus/$name/* + else + zip $OUT/${name}_seed_corpus.zip $SRC/nss-corpus/*/* + fi +} + +# Copy libFuzzer options +cp fuzz/options/*.options $OUT/ + +# Build the library (non-TLS fuzzing mode). +CXX="$CXX -stdlib=libc++" LDFLAGS="$CFLAGS" \ + ./build.sh -c -v --fuzz=oss --fuzz --disable-tests + +# Copy fuzzing targets. +for fuzzer in $(find ../dist/Debug/bin -name "nssfuzz-*" -printf "%f\n"); do + name=${fuzzer:8} + if [ -z "${disabled[$name]:-}" ]; then + [ -n "${tls_targets[$name]:-}" ] && name="${name}-no_fuzzer_mode" + copy_fuzzer $fuzzer $name + fi +done + +# Build the library again (TLS fuzzing mode). +CXX="$CXX -stdlib=libc++" LDFLAGS="$CFLAGS" \ + ./build.sh -c -v --fuzz=oss --fuzz=tls --disable-tests + +# Copy dual mode targets in TLS mode. +for name in "${!tls_targets[@]}"; do + if [ -z "${disabled[$name]:-}" ]; then + copy_fuzzer nssfuzz-$name $name + fi +done diff --git a/nss/automation/release/nspr-version.txt b/nss/automation/release/nspr-version.txt new file mode 100644 index 0000000..9a4d130 --- /dev/null +++ b/nss/automation/release/nspr-version.txt @@ -0,0 +1,10 @@ +4.15 + +# The first line of this file must contain the human readable NSPR +# version number, which is the minimum required version of NSPR +# that is supported by this version of NSS. +# +# This information is used by release automation, +# when creating an NSS source archive. +# +# All other lines in this file are ignored. diff --git a/nss/automation/release/nss-release-helper.py b/nss/automation/release/nss-release-helper.py index a7b4088..d168feb 100644..100755 --- a/nss/automation/release/nss-release-helper.py +++ b/nss/automation/release/nss-release-helper.py @@ -171,11 +171,13 @@ def set_4_digit_release_number(): set_all_lib_versions(version, major, minor, patch, build) def create_nss_release_archive(): - ensure_arguments_after_action(4, "nss_release_version nss_hg_release_tag nspr_release_version path_to_stage_directory") + ensure_arguments_after_action(3, "nss_release_version nss_hg_release_tag path_to_stage_directory") nssrel = args[1].strip() #e.g. 3.19.3 nssreltag = args[2].strip() #e.g. NSS_3_19_3_RTM - nsprrel = args[3].strip() #e.g. 4.10.8 - stagedir = args[4].strip() #e.g. ../stage + stagedir = args[3].strip() #e.g. ../stage + + with open('automation/release/nspr-version.txt') as nspr_version_file: + nsprrel = next(nspr_version_file).strip() nspr_tar = "nspr-" + nsprrel + ".tar.gz" nsprtar_with_path= stagedir + "/v" + nsprrel + "/src/" + nspr_tar diff --git a/nss/automation/taskcluster/docker-aarch64/Dockerfile b/nss/automation/taskcluster/docker-aarch64/Dockerfile new file mode 100644 index 0000000..2d7ade3 --- /dev/null +++ b/nss/automation/taskcluster/docker-aarch64/Dockerfile @@ -0,0 +1,30 @@ +FROM franziskus/xenial:aarch64 +MAINTAINER Franziskus Kiefer <franziskuskiefer@gmail.com> + +RUN useradd -d /home/worker -s /bin/bash -m worker +WORKDIR /home/worker + +# Add build and test scripts. +ADD bin /home/worker/bin +RUN chmod +x /home/worker/bin/* + +# Install dependencies. +ADD setup.sh /tmp/setup.sh +RUN bash /tmp/setup.sh + +# Change user. +# USER worker # See bug 1347473. + +# Env variables. +ENV HOME /home/worker +ENV SHELL /bin/bash +ENV USER worker +ENV LOGNAME worker +ENV HOSTNAME taskcluster-worker +ENV LANG en_US.UTF-8 +ENV LC_ALL en_US.UTF-8 +ENV HOST localhost +ENV DOMSUF localdomain + +# Set a default command for debugging. +CMD ["/bin/bash", "--login"] diff --git a/nss/automation/taskcluster/docker-aarch64/bin/checkout.sh b/nss/automation/taskcluster/docker-aarch64/bin/checkout.sh new file mode 100755 index 0000000..9167f6b --- /dev/null +++ b/nss/automation/taskcluster/docker-aarch64/bin/checkout.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -v -e -x + +if [ $(id -u) = 0 ]; then + # Drop privileges by re-running this script. + exec su worker $0 +fi + +# Default values for testing. +REVISION=${NSS_HEAD_REVISION:-default} +REPOSITORY=${NSS_HEAD_REPOSITORY:-https://hg.mozilla.org/projects/nss} + +# Clone NSS. +for i in 0 2 5; do + sleep $i + hg clone -r $REVISION $REPOSITORY nss && exit 0 + rm -rf nss +done +exit 1 diff --git a/nss/automation/taskcluster/docker-aarch64/setup.sh b/nss/automation/taskcluster/docker-aarch64/setup.sh new file mode 100755 index 0000000..b76514a --- /dev/null +++ b/nss/automation/taskcluster/docker-aarch64/setup.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -v -e -x + +export DEBIAN_FRONTEND=noninteractive + +apt-get -y update +apt-get -y install software-properties-common + +# Add more repos +add-apt-repository "deb http://ports.ubuntu.com/ xenial main restricted universe multiverse" +add-apt-repository "deb http://ports.ubuntu.com/ xenial-security main restricted universe multiverse" +add-apt-repository "deb http://ports.ubuntu.com/ xenial-updates main restricted universe multiverse" +add-apt-repository "deb http://ports.ubuntu.com/ xenial-backports main restricted universe multiverse" + +# Update. +apt-get -y update +apt-get -y dist-upgrade + +apt_packages=() +apt_packages+=('build-essential') +apt_packages+=('ca-certificates') +apt_packages+=('curl') +apt_packages+=('libxml2-utils') +apt_packages+=('zlib1g-dev') +apt_packages+=('ninja-build') +apt_packages+=('gyp') +apt_packages+=('mercurial') +apt_packages+=('locales') + +# Install packages. +apt-get install -y --no-install-recommends ${apt_packages[@]} + +locale-gen en_US.UTF-8 +dpkg-reconfigure locales + +# Cleanup. +rm -rf ~/.ccache ~/.cache +apt-get autoremove -y +apt-get clean +apt-get autoclean +rm $0 diff --git a/nss/automation/taskcluster/docker-arm/setup.sh b/nss/automation/taskcluster/docker-arm/setup.sh index 42d66a4..78c6392 100755 --- a/nss/automation/taskcluster/docker-arm/setup.sh +++ b/nss/automation/taskcluster/docker-arm/setup.sh @@ -12,6 +12,7 @@ apt_packages=() apt_packages+=('build-essential') apt_packages+=('ca-certificates') apt_packages+=('curl') +apt_packages+=('locales') apt_packages+=('python-dev') apt_packages+=('python-pip') apt_packages+=('python-setuptools') diff --git a/nss/automation/taskcluster/docker-clang-3.9/Dockerfile b/nss/automation/taskcluster/docker-clang-3.9/Dockerfile new file mode 100644 index 0000000..473ce64 --- /dev/null +++ b/nss/automation/taskcluster/docker-clang-3.9/Dockerfile @@ -0,0 +1,30 @@ +FROM ubuntu:16.04 +MAINTAINER Tim Taubert <ttaubert@mozilla.com> + +RUN useradd -d /home/worker -s /bin/bash -m worker +WORKDIR /home/worker + +# Add build and test scripts. +ADD bin /home/worker/bin +RUN chmod +x /home/worker/bin/* + +# Install dependencies. +ADD setup.sh /tmp/setup.sh +RUN bash /tmp/setup.sh + +# Change user. +USER worker + +# Env variables. +ENV HOME /home/worker +ENV SHELL /bin/bash +ENV USER worker +ENV LOGNAME worker +ENV HOSTNAME taskcluster-worker +ENV LANG en_US.UTF-8 +ENV LC_ALL en_US.UTF-8 +ENV HOST localhost +ENV DOMSUF localdomain + +# Set a default command for debugging. +CMD ["/bin/bash", "--login"] diff --git a/nss/automation/taskcluster/docker-clang-3.9/bin/checkout.sh b/nss/automation/taskcluster/docker-clang-3.9/bin/checkout.sh new file mode 100644 index 0000000..9167f6b --- /dev/null +++ b/nss/automation/taskcluster/docker-clang-3.9/bin/checkout.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -v -e -x + +if [ $(id -u) = 0 ]; then + # Drop privileges by re-running this script. + exec su worker $0 +fi + +# Default values for testing. +REVISION=${NSS_HEAD_REVISION:-default} +REPOSITORY=${NSS_HEAD_REPOSITORY:-https://hg.mozilla.org/projects/nss} + +# Clone NSS. +for i in 0 2 5; do + sleep $i + hg clone -r $REVISION $REPOSITORY nss && exit 0 + rm -rf nss +done +exit 1 diff --git a/nss/automation/taskcluster/docker-clang-3.9/setup.sh b/nss/automation/taskcluster/docker-clang-3.9/setup.sh new file mode 100644 index 0000000..7b7d534 --- /dev/null +++ b/nss/automation/taskcluster/docker-clang-3.9/setup.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +set -v -e -x + +# Update packages. +export DEBIAN_FRONTEND=noninteractive +apt-get -y update && apt-get -y upgrade + +# Need this to add keys for PPAs below. +apt-get install -y --no-install-recommends apt-utils + +apt_packages=() +apt_packages+=('ca-certificates') +apt_packages+=('curl') +apt_packages+=('locales') +apt_packages+=('xz-utils') + +# Latest Mercurial. +apt_packages+=('mercurial') +apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 41BD8711B1F0EC2B0D85B91CF59CE3A8323293EE +echo "deb http://ppa.launchpad.net/mercurial-ppa/releases/ubuntu xenial main" > /etc/apt/sources.list.d/mercurial.list + +# Install packages. +apt-get -y update +apt-get install -y --no-install-recommends ${apt_packages[@]} + +# Download clang. +curl -LO http://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz +curl -LO http://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz.sig +# Verify the signature. +gpg --keyserver pool.sks-keyservers.net --recv-keys B6C8F98282B944E3B0D5C2530FC3042E345AD05D +gpg --verify *.tar.xz.sig +# Install into /usr/local/. +tar xJvf *.tar.xz -C /usr/local --strip-components=1 +# Cleanup. +rm *.tar.xz* + +locale-gen en_US.UTF-8 +dpkg-reconfigure locales + +# Cleanup. +rm -rf ~/.ccache ~/.cache +apt-get autoremove -y +apt-get clean +apt-get autoclean +rm $0 diff --git a/nss/automation/taskcluster/docker-decision/setup.sh b/nss/automation/taskcluster/docker-decision/setup.sh index e5a6d20..5193852 100644 --- a/nss/automation/taskcluster/docker-decision/setup.sh +++ b/nss/automation/taskcluster/docker-decision/setup.sh @@ -7,7 +7,7 @@ export DEBIAN_FRONTEND=noninteractive apt-get -y update && apt-get -y upgrade # Need those to install newer packages below. -apt-get install -y --no-install-recommends apt-utils curl ca-certificates +apt-get install -y --no-install-recommends apt-utils curl ca-certificates locales # Latest Mercurial. apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 41BD8711B1F0EC2B0D85B91CF59CE3A8323293EE diff --git a/nss/automation/taskcluster/docker-fuzz/Dockerfile b/nss/automation/taskcluster/docker-fuzz/Dockerfile new file mode 100644 index 0000000..254f166 --- /dev/null +++ b/nss/automation/taskcluster/docker-fuzz/Dockerfile @@ -0,0 +1,33 @@ +FROM ubuntu:16.04 +MAINTAINER Tim Taubert <ttaubert@mozilla.com> + +RUN useradd -d /home/worker -s /bin/bash -m worker +WORKDIR /home/worker + +# Add build and test scripts. +ADD bin /home/worker/bin +RUN chmod +x /home/worker/bin/* + +# Install dependencies. +ADD setup.sh /tmp/setup.sh +RUN bash /tmp/setup.sh + +# Change user. +USER worker + +# Env variables. +ENV HOME /home/worker +ENV SHELL /bin/bash +ENV USER worker +ENV LOGNAME worker +ENV HOSTNAME taskcluster-worker +ENV LANG en_US.UTF-8 +ENV LC_ALL en_US.UTF-8 +ENV HOST localhost +ENV DOMSUF localdomain + +# LLVM 4.0 +ENV PATH "${PATH}:/home/worker/third_party/llvm-build/Release+Asserts/bin/" + +# Set a default command for debugging. +CMD ["/bin/bash", "--login"] diff --git a/nss/automation/taskcluster/docker-fuzz/bin/checkout.sh b/nss/automation/taskcluster/docker-fuzz/bin/checkout.sh new file mode 100644 index 0000000..9167f6b --- /dev/null +++ b/nss/automation/taskcluster/docker-fuzz/bin/checkout.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -v -e -x + +if [ $(id -u) = 0 ]; then + # Drop privileges by re-running this script. + exec su worker $0 +fi + +# Default values for testing. +REVISION=${NSS_HEAD_REVISION:-default} +REPOSITORY=${NSS_HEAD_REPOSITORY:-https://hg.mozilla.org/projects/nss} + +# Clone NSS. +for i in 0 2 5; do + sleep $i + hg clone -r $REVISION $REPOSITORY nss && exit 0 + rm -rf nss +done +exit 1 diff --git a/nss/automation/taskcluster/docker-fuzz/setup.sh b/nss/automation/taskcluster/docker-fuzz/setup.sh new file mode 100644 index 0000000..fcb7234 --- /dev/null +++ b/nss/automation/taskcluster/docker-fuzz/setup.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +set -v -e -x + +# Update packages. +export DEBIAN_FRONTEND=noninteractive +apt-get -y update && apt-get -y upgrade + +# Need this to add keys for PPAs below. +apt-get install -y --no-install-recommends apt-utils + +apt_packages=() +apt_packages+=('build-essential') +apt_packages+=('ca-certificates') +apt_packages+=('curl') +apt_packages+=('git') +apt_packages+=('gyp') +apt_packages+=('libssl-dev') +apt_packages+=('libxml2-utils') +apt_packages+=('locales') +apt_packages+=('ninja-build') +apt_packages+=('pkg-config') +apt_packages+=('zlib1g-dev') + +# 32-bit builds +apt_packages+=('gcc-multilib') +apt_packages+=('g++-multilib') + +# Latest Mercurial. +apt_packages+=('mercurial') +apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 41BD8711B1F0EC2B0D85B91CF59CE3A8323293EE +echo "deb http://ppa.launchpad.net/mercurial-ppa/releases/ubuntu xenial main" > /etc/apt/sources.list.d/mercurial.list + +# Install packages. +apt-get -y update +apt-get install -y --no-install-recommends ${apt_packages[@]} + +# 32-bit builds +dpkg --add-architecture i386 +apt-get -y update +apt-get install -y --no-install-recommends libssl-dev:i386 + +# Install LLVM/clang-4.0. +mkdir clang-tmp +git clone -n --depth 1 https://chromium.googlesource.com/chromium/src/tools/clang clang-tmp/clang +git -C clang-tmp/clang checkout HEAD scripts/update.py +clang-tmp/clang/scripts/update.py +rm -fr clang-tmp + +locale-gen en_US.UTF-8 +dpkg-reconfigure locales + +# Cleanup. +rm -rf ~/.ccache ~/.cache +apt-get autoremove -y +apt-get clean +apt-get autoclean +rm $0 diff --git a/nss/automation/taskcluster/docker/Dockerfile b/nss/automation/taskcluster/docker/Dockerfile index 35777c0..8a2256d 100644 --- a/nss/automation/taskcluster/docker/Dockerfile +++ b/nss/automation/taskcluster/docker/Dockerfile @@ -12,6 +12,9 @@ RUN chmod +x /home/worker/bin/* ADD setup.sh /tmp/setup.sh RUN bash /tmp/setup.sh +# Change user. +USER worker + # Env variables. ENV HOME /home/worker ENV SHELL /bin/bash @@ -23,5 +26,8 @@ ENV LC_ALL en_US.UTF-8 ENV HOST localhost ENV DOMSUF localdomain +# Rust + Go +ENV PATH "${PATH}:/home/worker/.cargo/bin/:/usr/lib/go-1.6/bin" + # Set a default command for debugging. CMD ["/bin/bash", "--login"] diff --git a/nss/automation/taskcluster/docker/setup.sh b/nss/automation/taskcluster/docker/setup.sh index f91c615..3ba4e85 100644 --- a/nss/automation/taskcluster/docker/setup.sh +++ b/nss/automation/taskcluster/docker/setup.sh @@ -16,6 +16,8 @@ apt_packages+=('curl') apt_packages+=('npm') apt_packages+=('git') apt_packages+=('golang-1.6') +apt_packages+=('libxml2-utils') +apt_packages+=('locales') apt_packages+=('ninja-build') apt_packages+=('pkg-config') apt_packages+=('zlib1g-dev') @@ -45,11 +47,19 @@ echo "deb http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu xenial main" > apt-get -y update apt-get install -y --no-install-recommends ${apt_packages[@]} -# 32-bit builds -ln -s /usr/include/x86_64-linux-gnu/zconf.h /usr/include +# Download clang. +curl -LO http://releases.llvm.org/4.0.0/clang+llvm-4.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz +curl -LO http://releases.llvm.org/4.0.0/clang+llvm-4.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz.sig +# Verify the signature. +gpg --keyserver pool.sks-keyservers.net --recv-keys B6C8F98282B944E3B0D5C2530FC3042E345AD05D +gpg --verify *.tar.xz.sig +# Install into /usr/local/. +tar xJvf *.tar.xz -C /usr/local --strip-components=1 +# Cleanup. +rm *.tar.xz* -# Install clang-3.9 into /usr/local/. -curl http://llvm.org/releases/3.9.0/clang+llvm-3.9.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz | tar xJv -C /usr/local --strip-components=1 +# Install latest Rust (stable). +su worker -c "curl https://sh.rustup.rs -sSf | sh -s -- -y" locale-gen en_US.UTF-8 dpkg-reconfigure locales diff --git a/nss/automation/taskcluster/graph/src/extend.js b/nss/automation/taskcluster/graph/src/extend.js index a6a8fcb..d9f6938 100644 --- a/nss/automation/taskcluster/graph/src/extend.js +++ b/nss/automation/taskcluster/graph/src/extend.js @@ -5,7 +5,20 @@ import merge from "./merge"; import * as queue from "./queue"; -const LINUX_IMAGE = {name: "linux", path: "automation/taskcluster/docker"}; +const LINUX_IMAGE = { + name: "linux", + path: "automation/taskcluster/docker" +}; + +const LINUX_CLANG39_IMAGE = { + name: "linux-clang-3.9", + path: "automation/taskcluster/docker-clang-3.9" +}; + +const FUZZ_IMAGE = { + name: "fuzz", + path: "automation/taskcluster/docker-fuzz" +}; const WINDOWS_CHECKOUT_CMD = "bash -c \"hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss || " + @@ -17,33 +30,45 @@ const WINDOWS_CHECKOUT_CMD = queue.filter(task => { if (task.group == "Builds") { // Remove extra builds on {A,UB}San and ARM. - if (task.collection == "asan" || task.collection == "arm-debug" || - task.collection == "gyp-asan") { + if (task.collection == "asan" || task.platform == "aarch64") { return false; } - // Remove extra builds w/o libpkix for non-linux64-debug. - if (task.symbol == "noLibpkix" && - (task.platform != "linux64" || task.collection != "debug")) { + // Make modular builds only on Linux make. + if (task.symbol == "modular" && task.collection != "make") { return false; } } - if (task.tests == "bogo") { - // No BoGo tests on Windows. - if (task.platform == "windows2012-64") { + if (task.tests == "bogo" || task.tests == "interop") { + // No windows + if (task.platform == "windows2012-64" || + task.platform == "windows2012-32") { return false; } - // No BoGo tests on ARM. - if (task.collection == "arm-debug") { + // No ARM; TODO: enable + if (task.platform == "aarch64") { return false; } } - // GYP builds with -Ddisable_libpkix=1 by default. - if ((task.collection == "gyp" || task.collection == "gyp-asan") && - task.tests == "chains") { + // Only old make builds have -Ddisable_libpkix=0 and can run chain tests. + if (task.tests == "chains" && task.collection != "make") { + return false; + } + + if (task.group == "Test") { + // Don't run test builds on old make platforms + if (task.collection == "make") { + return false; + } + } + + // Don't run additional hardware tests on ARM (we don't have anything there). + if (task.group == "Cipher" && task.platform == "aarch64" && task.env && + (task.env.NSS_DISABLE_PCLMUL == "1" || task.env.NSS_DISABLE_HW_AES == "1" + || task.env.NSS_DISABLE_AVX == "1")) { return false; } @@ -51,29 +76,18 @@ queue.filter(task => { }); queue.map(task => { - if (task.collection == "asan" || task.collection == "gyp-asan") { + if (task.collection == "asan") { // CRMF and FIPS tests still leak, unfortunately. if (task.tests == "crmf" || task.tests == "fips") { task.env.ASAN_OPTIONS = "detect_leaks=0"; } } - if (task.collection == "arm-debug") { - // These tests take quite some time on our poor ARM devices. - if (task.tests == "chains" || (task.tests == "ssl" && task.cycle == "standard")) { - task.maxRunTime = 14400; - } - } - // Windows is slow. if (task.platform == "windows2012-64" && task.tests == "chains") { task.maxRunTime = 7200; } - // Enable TLS 1.3 for every task. - task.env = task.env || {}; - task.env.NSS_ENABLE_TLS_1_3 = "1"; - return task; }); @@ -81,58 +95,48 @@ queue.map(task => { export default async function main() { await scheduleLinux("Linux 32 (opt)", { - env: {BUILD_OPT: "1"}, platform: "linux32", image: LINUX_IMAGE - }); + }, "-m32 --opt"); await scheduleLinux("Linux 32 (debug)", { platform: "linux32", collection: "debug", image: LINUX_IMAGE - }); + }, "-m32"); await scheduleLinux("Linux 64 (opt)", { - env: {USE_64: "1", BUILD_OPT: "1"}, platform: "linux64", image: LINUX_IMAGE - }); + }, "--opt"); await scheduleLinux("Linux 64 (debug)", { - env: {USE_64: "1"}, platform: "linux64", collection: "debug", image: LINUX_IMAGE }); - await scheduleLinux("Linux 64 (debug, gyp)", { + await scheduleLinux("Linux 64 (debug, make)", { + env: {USE_64: "1"}, + platform: "linux64", + image: LINUX_IMAGE, + collection: "make", command: [ - "/bin/bash", - "-c", - "bin/checkout.sh && nss/automation/taskcluster/scripts/build_gyp.sh" + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/build.sh" ], - platform: "linux64", - collection: "gyp", - image: LINUX_IMAGE }); - await scheduleLinux("Linux 64 (debug, gyp, asan, ubsan)", { + await scheduleLinux("Linux 32 (debug, make)", { + platform: "linux32", + image: LINUX_IMAGE, + collection: "make", command: [ - "/bin/bash", - "-c", - "bin/checkout.sh && nss/automation/taskcluster/scripts/build_gyp.sh -g -v --ubsan --asan" + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/build.sh" ], - env: { - ASAN_OPTIONS: "detect_odr_violation=0", // bug 1316276 - UBSAN_OPTIONS: "print_stacktrace=1", - NSS_DISABLE_ARENA_FREE_LIST: "1", - NSS_DISABLE_UNLOAD: "1", - CC: "clang", - CCC: "clang++" - }, - platform: "linux64", - collection: "gyp-asan", - image: LINUX_IMAGE }); await scheduleLinux("Linux 64 (ASan, debug)", { @@ -142,49 +146,87 @@ export default async function main() { NSS_DISABLE_UNLOAD: "1", CC: "clang", CCC: "clang++", - USE_UBSAN: "1", - USE_ASAN: "1", - USE_64: "1" }, platform: "linux64", collection: "asan", - image: LINUX_IMAGE - }); + image: LINUX_IMAGE, + features: ["allowPtrace"], + }, "--ubsan --asan"); + + await scheduleWindows("Windows 2012 64 (debug, make)", { + platform: "windows2012-64", + collection: "make", + env: {USE_64: "1"} + }, "build.sh"); + + await scheduleWindows("Windows 2012 32 (debug, make)", { + platform: "windows2012-32", + collection: "make" + }, "build.sh"); await scheduleWindows("Windows 2012 64 (opt)", { - env: {BUILD_OPT: "1"} - }); + platform: "windows2012-64", + }, "build_gyp.sh --opt"); await scheduleWindows("Windows 2012 64 (debug)", { + platform: "windows2012-64", collection: "debug" - }); + }, "build_gyp.sh"); - await scheduleFuzzing(); + await scheduleWindows("Windows 2012 32 (opt)", { + platform: "windows2012-32", + }, "build_gyp.sh --opt -m32"); - await scheduleTestBuilds(); + await scheduleWindows("Windows 2012 32 (debug)", { + platform: "windows2012-32", + collection: "debug" + }, "build_gyp.sh -m32"); + + await scheduleFuzzing(); + await scheduleFuzzing32(); await scheduleTools(); - await scheduleLinux("Linux 32 (ARM, debug)", { - image: "franziskus/nss-arm-ci", + let aarch64_base = { + image: "franziskus/nss-aarch64-ci", provisioner: "localprovisioner", - collection: "arm-debug", - workerType: "nss-rpi", - platform: "linux32", - maxRunTime: 7200, - tier: 3 - }); + workerType: "nss-aarch64", + platform: "aarch64", + maxRunTime: 7200 + }; + + await scheduleLinux("Linux AArch64 (debug)", + merge({ + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/build_gyp.sh" + ], + collection: "debug", + }, aarch64_base) + ); + + await scheduleLinux("Linux AArch64 (opt)", + merge({ + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/build_gyp.sh --opt" + ], + collection: "opt", + }, aarch64_base) + ); } /*****************************************************************************/ -async function scheduleLinux(name, base) { +async function scheduleLinux(name, base, args = "") { // Build base definition. let build_base = merge({ command: [ "/bin/bash", "-c", - "bin/checkout.sh && nss/automation/taskcluster/scripts/build.sh" + "bin/checkout.sh && nss/automation/taskcluster/scripts/build_gyp.sh " + args ], artifacts: { public: { @@ -224,12 +266,12 @@ async function scheduleLinux(name, base) { // Extra builds. let extra_base = merge({group: "Builds"}, build_base); queue.scheduleTask(merge(extra_base, { - name: `${name} w/ clang-3.9`, + name: `${name} w/ clang-4.0`, env: { CC: "clang", CCC: "clang++", }, - symbol: "clang-3.9" + symbol: "clang-4.0" })); queue.scheduleTask(merge(extra_base, { @@ -251,30 +293,54 @@ async function scheduleLinux(name, base) { })); queue.scheduleTask(merge(extra_base, { - name: `${name} w/ NSS_DISABLE_LIBPKIX=1`, - env: {NSS_DISABLE_LIBPKIX: "1"}, - symbol: "noLibpkix" + name: `${name} w/ modular builds`, + env: {NSS_BUILD_MODULAR: "1"}, + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/build.sh", + ], + symbol: "modular" })); + await scheduleTestBuilds(merge(base, {group: "Test"}), args); + return queue.submit(); } /*****************************************************************************/ +function scheduleFuzzingRun(base, name, target, max_len, symbol = null, corpus = null) { + const MAX_FUZZ_TIME = 300; + + queue.scheduleTask(merge(base, { + name, + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/fuzz.sh " + + `${target} nss/fuzz/corpus/${corpus || target} ` + + `-max_total_time=${MAX_FUZZ_TIME} ` + + `-max_len=${max_len}` + ], + symbol: symbol || name + })); +} + async function scheduleFuzzing() { let base = { env: { - // bug 1316276 - ASAN_OPTIONS: "allocator_may_return_null=1:detect_odr_violation=0", + ASAN_OPTIONS: "allocator_may_return_null=1:detect_stack_use_after_return=1", UBSAN_OPTIONS: "print_stacktrace=1", NSS_DISABLE_ARENA_FREE_LIST: "1", NSS_DISABLE_UNLOAD: "1", CC: "clang", CCC: "clang++" }, + features: ["allowPtrace"], platform: "linux64", collection: "fuzz", - image: LINUX_IMAGE + image: FUZZ_IMAGE }; // Build base definition. @@ -301,9 +367,22 @@ async function scheduleFuzzing() { name: "Linux x64 (debug, fuzz)" })); + // The task that builds NSPR+NSS (TLS fuzzing mode). + let task_build_tls = queue.scheduleTask(merge(build_base, { + name: "Linux x64 (debug, TLS fuzz)", + symbol: "B", + group: "TLS", + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && " + + "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz=tls" + ], + })); + // Schedule tests. queue.scheduleTask(merge(base, { - parent: task_build, + parent: task_build_tls, name: "Gtests", command: [ "/bin/bash", @@ -317,56 +396,155 @@ async function scheduleFuzzing() { kind: "test" })); - queue.scheduleTask(merge(base, { - parent: task_build, - name: "Cert", + // Schedule fuzzing runs. + let run_base = merge(base, {parent: task_build, kind: "test"}); + scheduleFuzzingRun(run_base, "CertDN", "certDN", 4096); + scheduleFuzzingRun(run_base, "QuickDER", "quickder", 10000); + + // Schedule MPI fuzzing runs. + let mpi_base = merge(run_base, {group: "MPI"}); + let mpi_names = ["add", "addmod", "div", "expmod", "mod", "mulmod", "sqr", + "sqrmod", "sub", "submod"]; + for (let name of mpi_names) { + scheduleFuzzingRun(mpi_base, `MPI (${name})`, `mpi-${name}`, 4096, name); + } + scheduleFuzzingRun(mpi_base, `MPI (invmod)`, `mpi-invmod`, 256, "invmod"); + + // Schedule TLS fuzzing runs (non-fuzzing mode). + let tls_base = merge(run_base, {group: "TLS"}); + scheduleFuzzingRun(tls_base, "TLS Client", "tls-client", 20000, "client-nfm", + "tls-client-no_fuzzer_mode"); + scheduleFuzzingRun(tls_base, "TLS Server", "tls-server", 20000, "server-nfm", + "tls-server-no_fuzzer_mode"); + scheduleFuzzingRun(tls_base, "DTLS Client", "dtls-client", 20000, + "dtls-client-nfm", "dtls-client-no_fuzzer_mode"); + scheduleFuzzingRun(tls_base, "DTLS Server", "dtls-server", 20000, + "dtls-server-nfm", "dtls-server-no_fuzzer_mode"); + + // Schedule TLS fuzzing runs (fuzzing mode). + let tls_fm_base = merge(tls_base, {parent: task_build_tls}); + scheduleFuzzingRun(tls_fm_base, "TLS Client", "tls-client", 20000, "client"); + scheduleFuzzingRun(tls_fm_base, "TLS Server", "tls-server", 20000, "server"); + scheduleFuzzingRun(tls_fm_base, "DTLS Client", "dtls-client", 20000, "dtls-client"); + scheduleFuzzingRun(tls_fm_base, "DTLS Server", "dtls-server", 20000, "dtls-server"); + + return queue.submit(); +} + +async function scheduleFuzzing32() { + let base = { + env: { + ASAN_OPTIONS: "allocator_may_return_null=1:detect_stack_use_after_return=1", + UBSAN_OPTIONS: "print_stacktrace=1", + NSS_DISABLE_ARENA_FREE_LIST: "1", + NSS_DISABLE_UNLOAD: "1", + CC: "clang", + CCC: "clang++" + }, + features: ["allowPtrace"], + platform: "linux32", + collection: "fuzz", + image: FUZZ_IMAGE + }; + + // Build base definition. + let build_base = merge({ command: [ "/bin/bash", "-c", - "bin/checkout.sh && nss/automation/taskcluster/scripts/fuzz.sh " + - "cert nss/fuzz/corpus/cert -max_total_time=300" + "bin/checkout.sh && " + + "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz -m32" ], - // Need a privileged docker container to remove this. - env: {ASAN_OPTIONS: "detect_leaks=0"}, - symbol: "SCert", - kind: "test" + artifacts: { + public: { + expires: 24 * 7, + type: "directory", + path: "/home/worker/artifacts" + } + }, + kind: "build", + symbol: "B" + }, base); + + // The task that builds NSPR+NSS. + let task_build = queue.scheduleTask(merge(build_base, { + name: "Linux 32 (debug, fuzz)" })); + // The task that builds NSPR+NSS (TLS fuzzing mode). + let task_build_tls = queue.scheduleTask(merge(build_base, { + name: "Linux 32 (debug, TLS fuzz)", + symbol: "B", + group: "TLS", + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && " + + "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz=tls -m32" + ], + })); + + // Schedule tests. queue.scheduleTask(merge(base, { - parent: task_build, - name: "SPKI", + parent: task_build_tls, + name: "Gtests", command: [ "/bin/bash", "-c", - "bin/checkout.sh && nss/automation/taskcluster/scripts/fuzz.sh " + - "spki nss/fuzz/corpus/spki -max_total_time=300" + "bin/checkout.sh && nss/automation/taskcluster/scripts/run_tests.sh" ], - // Need a privileged docker container to remove this. - env: {ASAN_OPTIONS: "detect_leaks=0"}, - symbol: "SPKI", + env: {GTESTFILTER: "*Fuzz*"}, + tests: "ssl_gtests gtests", + cycle: "standard", + symbol: "Gtest", kind: "test" })); + // Schedule fuzzing runs. + let run_base = merge(base, {parent: task_build, kind: "test"}); + scheduleFuzzingRun(run_base, "CertDN", "certDN", 4096); + scheduleFuzzingRun(run_base, "QuickDER", "quickder", 10000); + + // Schedule MPI fuzzing runs. + let mpi_base = merge(run_base, {group: "MPI"}); + let mpi_names = ["add", "addmod", "div", "expmod", "mod", "mulmod", "sqr", + "sqrmod", "sub", "submod"]; + for (let name of mpi_names) { + scheduleFuzzingRun(mpi_base, `MPI (${name})`, `mpi-${name}`, 4096, name); + } + scheduleFuzzingRun(mpi_base, `MPI (invmod)`, `mpi-invmod`, 256, "invmod"); + + // Schedule TLS fuzzing runs (non-fuzzing mode). + let tls_base = merge(run_base, {group: "TLS"}); + scheduleFuzzingRun(tls_base, "TLS Client", "tls-client", 20000, "client-nfm", + "tls-client-no_fuzzer_mode"); + scheduleFuzzingRun(tls_base, "TLS Server", "tls-server", 20000, "server-nfm", + "tls-server-no_fuzzer_mode"); + scheduleFuzzingRun(tls_base, "DTLS Client", "dtls-client", 20000, + "dtls-client-nfm", "dtls-client-no_fuzzer_mode"); + scheduleFuzzingRun(tls_base, "DTLS Server", "dtls-server", 20000, + "dtls-server-nfm", "dtls-server-no_fuzzer_mode"); + + // Schedule TLS fuzzing runs (fuzzing mode). + let tls_fm_base = merge(tls_base, {parent: task_build_tls}); + scheduleFuzzingRun(tls_fm_base, "TLS Client", "tls-client", 20000, "client"); + scheduleFuzzingRun(tls_fm_base, "TLS Server", "tls-server", 20000, "server"); + scheduleFuzzingRun(tls_fm_base, "DTLS Client", "dtls-client", 20000, "dtls-client"); + scheduleFuzzingRun(tls_fm_base, "DTLS Server", "dtls-server", 20000, "dtls-server"); + return queue.submit(); } /*****************************************************************************/ -async function scheduleTestBuilds() { - let base = { - platform: "linux64", - collection: "gyp", - group: "Test", - image: LINUX_IMAGE - }; - +async function scheduleTestBuilds(base, args = "") { // Build base definition. let build = merge({ command: [ "/bin/bash", "-c", "bin/checkout.sh && " + - "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --test" + "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --test --ct-verif " + args ], artifacts: { public: { @@ -377,7 +555,7 @@ async function scheduleTestBuilds() { }, kind: "build", symbol: "B", - name: "Linux 64 (debug, gyp, test)" + name: "Linux 64 (debug, test)" }, base); // The task that builds NSPR+NSS. @@ -397,6 +575,19 @@ async function scheduleTestBuilds() { symbol: "mpi", kind: "test" })); + queue.scheduleTask(merge(base, { + parent: task_build, + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/run_tests.sh" + ], + name: "Gtests", + symbol: "Gtest", + tests: "gtests", + cycle: "standard", + kind: "test" + })); return queue.submit(); } @@ -404,10 +595,9 @@ async function scheduleTestBuilds() { /*****************************************************************************/ -async function scheduleWindows(name, base) { +async function scheduleWindows(name, base, build_script) { base = merge(base, { workerType: "nss-win2012r2", - platform: "windows2012-64", env: { PATH: "c:\\mozilla-build\\python;c:\\mozilla-build\\msys\\local\\bin;" + "c:\\mozilla-build\\7zip;c:\\mozilla-build\\info-zip;" + @@ -417,7 +607,6 @@ async function scheduleWindows(name, base) { "c:\\mozilla-build\\wget", DOMSUF: "localdomain", HOST: "localhost", - USE_64: "1" } }); @@ -425,7 +614,7 @@ async function scheduleWindows(name, base) { let build_base = merge(base, { command: [ WINDOWS_CHECKOUT_CMD, - "bash -c nss/automation/taskcluster/windows/build.sh" + `bash -c 'nss/automation/taskcluster/windows/${build_script}'` ], artifacts: [{ expires: 24 * 7, @@ -475,10 +664,25 @@ function scheduleTests(task_build, task_cert, test_base) { name: "Bogo tests", symbol: "Bogo", tests: "bogo", cycle: "standard" })); queue.scheduleTask(merge(no_cert_base, { + name: "Interop tests", symbol: "Interop", tests: "interop", cycle: "standard" + })); + queue.scheduleTask(merge(no_cert_base, { name: "Chains tests", symbol: "Chains", tests: "chains" })); queue.scheduleTask(merge(no_cert_base, { - name: "Cipher tests", symbol: "Cipher", tests: "cipher" + name: "Cipher tests", symbol: "Default", tests: "cipher", group: "Cipher" + })); + queue.scheduleTask(merge(no_cert_base, { + name: "Cipher tests", symbol: "NoAESNI", tests: "cipher", + env: {NSS_DISABLE_HW_AES: "1"}, group: "Cipher" + })); + queue.scheduleTask(merge(no_cert_base, { + name: "Cipher tests", symbol: "NoPCLMUL", tests: "cipher", + env: {NSS_DISABLE_PCLMUL: "1"}, group: "Cipher" + })); + queue.scheduleTask(merge(no_cert_base, { + name: "Cipher tests", symbol: "NoAVX", tests: "cipher", + env: {NSS_DISABLE_AVX: "1"}, group: "Cipher" })); queue.scheduleTask(merge(no_cert_base, { name: "EC tests", symbol: "EC", tests: "ec" @@ -531,7 +735,6 @@ function scheduleTests(task_build, task_cert, test_base) { async function scheduleTools() { let base = { - image: LINUX_IMAGE, platform: "nss-tools", kind: "test" }; @@ -539,6 +742,7 @@ async function scheduleTools() { queue.scheduleTask(merge(base, { symbol: "clang-format-3.9", name: "clang-format-3.9", + image: LINUX_CLANG39_IMAGE, command: [ "/bin/bash", "-c", @@ -547,8 +751,9 @@ async function scheduleTools() { })); queue.scheduleTask(merge(base, { - symbol: "scan-build-3.9", - name: "scan-build-3.9", + symbol: "scan-build-4.0", + name: "scan-build-4.0", + image: LINUX_IMAGE, env: { USE_64: "1", CC: "clang", diff --git a/nss/automation/taskcluster/graph/src/queue.js b/nss/automation/taskcluster/graph/src/queue.js index 2a4a7b3..b78c54d 100644 --- a/nss/automation/taskcluster/graph/src/queue.js +++ b/nss/automation/taskcluster/graph/src/queue.js @@ -80,6 +80,7 @@ function parseTreeherder(def) { } function convertTask(def) { + let scopes = []; let dependencies = []; let env = merge({ @@ -110,12 +111,16 @@ function convertTask(def) { payload.image = def.image; } + if (def.artifacts) { + payload.artifacts = parseArtifacts(def.artifacts); + } + if (def.features) { payload.features = parseFeatures(def.features); - } - if (def.artifacts) { - payload.artifacts = parseArtifacts(def.artifacts); + if (payload.features.allowPtrace) { + scopes.push("docker-worker:feature:allowPtrace"); + } } return { @@ -123,6 +128,7 @@ function convertTask(def) { workerType: def.workerType || "hg-worker", schedulerId: "task-graph-scheduler", + scopes, created: fromNow(0), deadline: fromNow(24), diff --git a/nss/automation/taskcluster/graph/src/try_syntax.js b/nss/automation/taskcluster/graph/src/try_syntax.js index 695c9e9..7748e06 100644 --- a/nss/automation/taskcluster/graph/src/try_syntax.js +++ b/nss/automation/taskcluster/graph/src/try_syntax.js @@ -22,8 +22,10 @@ function parseOptions(opts) { } // Parse platforms. - let allPlatforms = ["linux", "linux64", "linux64-asan", "win64", "arm", - "linux64-gyp", "linux64-gyp-asan", "linux64-fuzz"]; + let allPlatforms = ["linux", "linux64", "linux64-asan", + "win", "win64", "win-make", "win64-make", + "linux64-make", "linux-make", "linux-fuzz", + "linux64-fuzz", "aarch64"]; let platforms = intersect(opts.platform.split(/\s*,\s*/), allPlatforms); // If the given value is nonsense or "none" default to all platforms. @@ -34,7 +36,7 @@ function parseOptions(opts) { // Parse unit tests. let aliases = {"gtests": "gtest"}; let allUnitTests = ["bogo", "crmf", "chains", "cipher", "db", "ec", "fips", - "gtest", "lowhash", "merge", "sdr", "smime", "tools", + "gtest", "interop", "lowhash", "merge", "sdr", "smime", "tools", "ssl", "mpi", "scert", "spki"]; let unittests = intersect(opts.unittests.split(/\s*,\s*/).map(t => { return aliases[t] || t; @@ -82,11 +84,13 @@ function filter(opts) { // Filter unit tests. if (task.tests) { let found = opts.unittests.some(test => { - // TODO: think of something more intelligent here. - if (task.symbol.toLowerCase().startsWith("mpi") && test == "mpi") { + if (task.group && task.group.toLowerCase() == "ssl" && test == "ssl") { return true; } - return (task.group || task.symbol).toLowerCase().startsWith(test); + if (task.group && task.group.toLowerCase() == "cipher" && test == "cipher") { + return true; + } + return task.symbol.toLowerCase().startsWith(test); }); if (!found) { @@ -105,12 +109,15 @@ function filter(opts) { let found = opts.platforms.some(platform => { let aliases = { "linux": "linux32", + "linux-fuzz": "linux32", "linux64-asan": "linux64", "linux64-fuzz": "linux64", - "linux64-gyp": "linux64", - "linux64-gyp-asan": "linux64", + "linux64-make": "linux64", + "linux-make": "linux32", + "win64-make": "windows2012-64", + "win-make": "windows2012-32", "win64": "windows2012-64", - "arm": "linux32" + "win": "windows2012-32" }; // Check the platform name. @@ -119,13 +126,10 @@ function filter(opts) { // Additional checks. if (platform == "linux64-asan") { keep &= coll("asan"); - } else if (platform == "arm") { - keep &= coll("arm-opt") || coll("arm-debug"); - } else if (platform == "linux64-gyp") { - keep &= coll("gyp"); - } else if (platform == "linux64-gyp-asan") { - keep &= coll("gyp-asan"); - } else if (platform == "linux64-fuzz") { + } else if (platform == "linux64-make" || platform == "linux-make" || + platform == "win64-make" || platform == "win-make") { + keep &= coll("make"); + } else if (platform == "linux64-fuzz" || platform == "linux-fuzz") { keep &= coll("fuzz"); } else { keep &= coll("opt") || coll("debug"); @@ -139,8 +143,8 @@ function filter(opts) { } // Finally, filter by build type. - let isDebug = coll("debug") || coll("asan") || coll("arm-debug") || - coll("gyp") || coll("fuzz"); + let isDebug = coll("debug") || coll("asan") || coll("make") || + coll("fuzz"); return (isDebug && opts.builds.includes("d")) || (!isDebug && opts.builds.includes("o")); } diff --git a/nss/automation/taskcluster/scripts/build.sh b/nss/automation/taskcluster/scripts/build.sh index 69968b1..649fdaa 100755 --- a/nss/automation/taskcluster/scripts/build.sh +++ b/nss/automation/taskcluster/scripts/build.sh @@ -1,14 +1,17 @@ #!/usr/bin/env bash -source $(dirname $0)/tools.sh +source $(dirname "$0")/tools.sh -if [[ $(id -u) -eq 0 ]]; then - # Drop privileges by re-running this script. - exec su worker $0 +if [ -n "$NSS_BUILD_MODULAR" ]; then + $(dirname "$0")/build_nspr.sh || exit $? + $(dirname "$0")/build_util.sh || exit $? + $(dirname "$0")/build_softoken.sh || exit $? + $(dirname "$0")/build_nss.sh || exit $? + exit fi # Clone NSPR if needed. -hg_clone https://hg.mozilla.org/projects/nspr nspr default +hg_clone https://hg.mozilla.org/projects/nspr ./nspr default # Build. make -C nss nss_build_all diff --git a/nss/automation/taskcluster/scripts/build_gyp.sh b/nss/automation/taskcluster/scripts/build_gyp.sh index 590e634..7190bd5 100755 --- a/nss/automation/taskcluster/scripts/build_gyp.sh +++ b/nss/automation/taskcluster/scripts/build_gyp.sh @@ -1,17 +1,12 @@ #!/usr/bin/env bash -source $(dirname $0)/tools.sh - -if [[ $(id -u) -eq 0 ]]; then - # Drop privileges by re-running this script. - exec su worker -c "$0 $*" -fi +source $(dirname "$0")/tools.sh # Clone NSPR if needed. -hg_clone https://hg.mozilla.org/projects/nspr nspr default +hg_clone https://hg.mozilla.org/projects/nspr ./nspr default # Build. -nss/build.sh ${*--g -v} +nss/build.sh -g -v "$@" # Package. mkdir artifacts diff --git a/nss/automation/taskcluster/scripts/build_nspr.sh b/nss/automation/taskcluster/scripts/build_nspr.sh new file mode 100755 index 0000000..4d19034 --- /dev/null +++ b/nss/automation/taskcluster/scripts/build_nspr.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -v -e -x + +source $(dirname $0)/tools.sh + +# Clone NSPR if needed. +hg_clone https://hg.mozilla.org/projects/nspr nspr default + +# Build. +rm -rf dist +make -C nss build_nspr + +# Package. +test -d artifacts || mkdir artifacts +rm -rf dist-nspr +mv dist dist-nspr +tar cvfjh artifacts/dist-nspr.tar.bz2 dist-nspr diff --git a/nss/automation/taskcluster/scripts/build_nss.sh b/nss/automation/taskcluster/scripts/build_nss.sh new file mode 100755 index 0000000..b909bc3 --- /dev/null +++ b/nss/automation/taskcluster/scripts/build_nss.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +set -v -e -x + +source $(dirname $0)/tools.sh +source $(dirname $0)/split.sh + +test -d dist-softoken || { echo "run build_softoken.sh first" 1>&2; exit 1; } + +rm -rf nss-nss +split_nss nss nss-nss + +# Build. +export NSS_BUILD_WITHOUT_SOFTOKEN=1 +export NSS_USE_SYSTEM_FREEBL=1 + +platform=`make -s -C nss platform` + +export NSPR_LIB_DIR="$PWD/dist-nspr/$platform/lib" +export NSSUTIL_LIB_DIR="$PWD/dist-util/$platform/lib" +export FREEBL_LIB_DIR="$PWD/dist-softoken/$platform/lib" +export SOFTOKEN_LIB_DIR="$PWD/dist-softoken/$platform/lib" +export FREEBL_LIBS=-lfreebl + +export NSS_NO_PKCS11_BYPASS=1 +export FREEBL_NO_DEPEND=1 + +export LIBRARY_PATH="$PWD/dist-nspr/$platform/lib:$PWD/dist-util/$platform/lib:$PWD/dist-softoken/$platform/lib" +export LD_LIBRARY_PATH="$LIBRARY_PATH:$LD_LIBRARY_PATH" +export INCLUDES="-I$PWD/dist-nspr/$platform/include -I$PWD/dist-util/public/nss -I$PWD/dist-softoken/public/nss" + +rm -rf dist +make -C nss-nss nss_build_all + +# Package. +test -d artifacts || mkdir artifacts +rm -rf dist-nss +mv dist dist-nss +tar cvfjh artifacts/dist-nss.tar.bz2 dist-nss diff --git a/nss/automation/taskcluster/scripts/build_softoken.sh b/nss/automation/taskcluster/scripts/build_softoken.sh new file mode 100755 index 0000000..e5aaecc --- /dev/null +++ b/nss/automation/taskcluster/scripts/build_softoken.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -v -e -x + +source $(dirname $0)/tools.sh +source $(dirname $0)/split.sh + +test -d dist-util || { echo "run build_util.sh first" 1>&2; exit 1; } + +rm -rf nss-softoken +split_softoken nss nss-softoken + +# Build. +platform=`make -s -C nss platform` +export LIBRARY_PATH="$PWD/dist-nspr/$platform/lib:$PWD/dist-util/$platform/lib" +export LD_LIBRARY_PATH="$LIBRARY_PATH:$LD_LIBRARY_PATH" +export INCLUDES="-I$PWD/dist-nspr/$platform/include -I$PWD/dist-util/public/nss" +export NSS_BUILD_SOFTOKEN_ONLY=1 + +rm -rf dist +make -C nss-softoken nss_build_all + +mv dist/private/nss/blapi.h dist/public/nss +mv dist/private/nss/alghmac.h dist/public/nss + +# Package. +test -d artifacts || mkdir artifacts +rm -rf dist-softoken +mv dist dist-softoken +tar cvfjh artifacts/dist-softoken.tar.bz2 dist-softoken diff --git a/nss/automation/taskcluster/scripts/build_util.sh b/nss/automation/taskcluster/scripts/build_util.sh new file mode 100755 index 0000000..0d2ecc5 --- /dev/null +++ b/nss/automation/taskcluster/scripts/build_util.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -v -e -x + +source $(dirname $0)/tools.sh +source $(dirname $0)/split.sh + +rm -rf nss-util +split_util nss nss-util + +# Build. +platform=`make -s -C nss platform` +export LIBRARY_PATH="$PWD/dist-nspr/$platform/lib" +export LD_LIBRARY_PATH="$LIBRARY_PATH:$LD_LIBRARY_PATH" +export INCLUDES="-I$PWD/dist-nspr/$platform/include" +export NSS_BUILD_UTIL_ONLY=1 + +rm -rf dist +make -C nss-util nss_build_all + +# Package. +test -d artifacts || mkdir artifacts +rm -rf dist-util +mv dist dist-util +tar cvfjh artifacts/dist-util.tar.bz2 dist-util diff --git a/nss/automation/taskcluster/scripts/extend_task_graph.sh b/nss/automation/taskcluster/scripts/extend_task_graph.sh index 5a3fb8d..ade84cd 100755 --- a/nss/automation/taskcluster/scripts/extend_task_graph.sh +++ b/nss/automation/taskcluster/scripts/extend_task_graph.sh @@ -1,11 +1,6 @@ #!/usr/bin/env bash -set -v -e -x - -if [ $(id -u) = 0 ]; then - # Drop privileges by re-running this script. - exec su worker $0 -fi +source $(dirname "$0")/tools.sh mkdir -p /home/worker/artifacts diff --git a/nss/automation/taskcluster/scripts/fuzz.sh b/nss/automation/taskcluster/scripts/fuzz.sh index 5f8dd7b..75851ff 100755 --- a/nss/automation/taskcluster/scripts/fuzz.sh +++ b/nss/automation/taskcluster/scripts/fuzz.sh @@ -1,20 +1,32 @@ #!/usr/bin/env bash -source $(dirname $0)/tools.sh +source $(dirname "$0")/tools.sh -if [ $(id -u) = 0 ]; then - # Drop privileges by re-running this script. - exec su worker -c "$0 $*" -fi +type="$1" +shift # Fetch artifact if needed. fetch_dist # Clone corpus. -./nss/fuzz/clone_corpus.sh +./nss/fuzz/config/clone_corpus.sh + +# Ensure we have a corpus. +if [ ! -d "nss/fuzz/corpus/$type" ]; then + mkdir -p nss/fuzz/corpus/$type + + set +x + + # Create a corpus out of what we have. + for f in $(find nss/fuzz/corpus -type f); do + cp $f "nss/fuzz/corpus/$type" + done + + set -x +fi # Fetch objdir name. objdir=$(cat dist/latest) # Run nssfuzz. -LD_LIBRARY_PATH=$LD_LIBRARY_PATH:dist/$objdir/lib dist/$objdir/bin/nssfuzz $* +dist/$objdir/bin/nssfuzz-"$type" "$@" diff --git a/nss/automation/taskcluster/scripts/gen_certs.sh b/nss/automation/taskcluster/scripts/gen_certs.sh index aee1001..b8d4f60 100755 --- a/nss/automation/taskcluster/scripts/gen_certs.sh +++ b/nss/automation/taskcluster/scripts/gen_certs.sh @@ -1,16 +1,6 @@ #!/usr/bin/env bash -set -v -e -x - -source $(dirname $0)/tools.sh - -if [ $(id -u) = 0 ]; then - # Stupid Docker. - echo "127.0.0.1 localhost.localdomain" >> /etc/hosts - - # Drop privileges by re-running this script. - exec su worker $0 -fi +source $(dirname "$0")/tools.sh # Fetch artifact if needed. fetch_dist diff --git a/nss/automation/taskcluster/scripts/run_clang_format.sh b/nss/automation/taskcluster/scripts/run_clang_format.sh index c4b6029..05e6317 100755 --- a/nss/automation/taskcluster/scripts/run_clang_format.sh +++ b/nss/automation/taskcluster/scripts/run_clang_format.sh @@ -1,56 +1,57 @@ #!/usr/bin/env bash -set -v -e -x +source $(dirname "$0")/tools.sh -if [ $(id -u) -eq 0 ]; then - # Drop privileges by re-running this script. - exec su worker $0 "$@" -fi +set +x # Apply clang-format on the provided folder and verify that this doesn't change any file. # If any file differs after formatting, the script eventually exits with 1. # Any differences between formatted and unformatted files is printed to stdout to give a hint what's wrong. -# Includes a default set of directories. +# Includes a default set of directories NOT to clang-format on. +blacklist=( + "./automation" \ + "./coreconf" \ + "./doc" \ + "./pkg" \ + "./tests" \ + "./lib/libpkix" \ + "./lib/zlib" \ + "./lib/sqlite" \ + "./gtests/google_test" \ + "./.hg" \ +) + +top="$PWD/$(dirname $0)/../../.." +cd "$top" if [ $# -gt 0 ]; then dirs=("$@") else - top=$(dirname $0)/../../.. - dirs=( \ - "$top/cmd" \ - "$top/fuzz" \ - "$top/lib/base" \ - "$top/lib/certdb" \ - "$top/lib/certhigh" \ - "$top/lib/ckfw" \ - "$top/lib/crmf" \ - "$top/lib/cryptohi" \ - "$top/lib/dbm" \ - "$top/lib/dev" \ - "$top/lib/freebl" \ - "$top/lib/jar" \ - "$top/lib/nss" \ - "$top/lib/pk11wrap" \ - "$top/lib/pkcs7" \ - "$top/lib/pkcs12" \ - "$top/lib/pki" \ - "$top/lib/smime" \ - "$top/lib/softoken" \ - "$top/lib/ssl" \ - "$top/lib/sysinit" \ - "$top/lib/util" \ - "$top/gtests/common" \ - "$top/gtests/der_gtest" \ - "$top/gtests/freebl_gtest" \ - "$top/gtests/pk11_gtest" \ - "$top/gtests/ssl_gtest" \ - "$top/gtests/util_gtest" \ - ) + dirs=($(find . ! -path . \( ! -regex '.*/' \) -maxdepth 2 -mindepth 1 -type d)) fi +format_folder() +{ + for black in "${blacklist[@]}"; do + if [[ "$1" == "$black"* ]]; then + echo "skip $1" + return 1 + fi + done + return 0 +} + for dir in "${dirs[@]}"; do - find "$dir" -type f \( -name '*.[ch]' -o -name '*.cc' \) -exec clang-format -i {} \+ + if format_folder "$dir" ; then + c="${dir//[^\/]}" + echo "formatting $dir ..." + depth="" + if [ "${#c}" == "1" ]; then + depth="-maxdepth 1" + fi + find "$dir" $depth -type f \( -name '*.[ch]' -o -name '*.cc' \) -exec clang-format -i {} \+ + fi done TMPFILE=$(mktemp /tmp/$(basename $0).XXXXXX) diff --git a/nss/automation/taskcluster/scripts/run_scan_build.sh b/nss/automation/taskcluster/scripts/run_scan_build.sh index 99f80ab..4024c22 100755 --- a/nss/automation/taskcluster/scripts/run_scan_build.sh +++ b/nss/automation/taskcluster/scripts/run_scan_build.sh @@ -1,15 +1,10 @@ #!/usr/bin/env bash -source $(dirname $0)/tools.sh - -if [ $(id -u) = 0 ]; then - # Drop privileges by re-running this script. - exec su worker $0 $@ -fi +source $(dirname "$0")/tools.sh # Clone NSPR if needed. if [ ! -d "nspr" ]; then - hg_clone https://hg.mozilla.org/projects/nspr nspr default + hg_clone https://hg.mozilla.org/projects/nspr ./nspr default fi # Build. diff --git a/nss/automation/taskcluster/scripts/run_tests.sh b/nss/automation/taskcluster/scripts/run_tests.sh index 4c87e7e..b8e2676 100755 --- a/nss/automation/taskcluster/scripts/run_tests.sh +++ b/nss/automation/taskcluster/scripts/run_tests.sh @@ -1,14 +1,6 @@ #!/usr/bin/env bash -source $(dirname $0)/tools.sh - -if [ $(id -u) = 0 ]; then - # Stupid Docker. - echo "127.0.0.1 localhost.localdomain" >> /etc/hosts - - # Drop privileges by re-running this script. - exec su worker $0 -fi +source $(dirname "$0")/tools.sh # Fetch artifact if needed. fetch_dist diff --git a/nss/automation/taskcluster/scripts/split.sh b/nss/automation/taskcluster/scripts/split.sh new file mode 100644 index 0000000..c43c3e4 --- /dev/null +++ b/nss/automation/taskcluster/scripts/split.sh @@ -0,0 +1,152 @@ +copy_top() +{ + srcdir_="$1" + dstdir_="$2" + files=`find "$srcdir_" -maxdepth 1 -mindepth 1 -type f` + for f in $files; do + cp -p "$f" "$dstdir_" + done +} + +split_util() { + nssdir="$1" + dstdir="$2" + + # Prepare a source tree only containing files to build nss-util: + # + # nss/dbm full directory + # nss/coreconf full directory + # nss top files only + # nss/lib top files only + # nss/lib/util full directory + + # Copy everything. + cp -R $nssdir $dstdir + + # Skip gtests when building. + sed '/^DIRS = /s/ gtests$//' $nssdir/manifest.mn > $dstdir/manifest.mn-t && mv $dstdir/manifest.mn-t $dstdir/manifest.mn + + # Remove subdirectories that we don't want. + rm -rf $dstdir/cmd + rm -rf $dstdir/tests + rm -rf $dstdir/lib + rm -rf $dstdir/automation + rm -rf $dstdir/gtests + rm -rf $dstdir/doc + + # Start with an empty cmd lib directories to be filled selectively. + mkdir $dstdir/cmd + cp $nssdir/cmd/Makefile $dstdir/cmd + cp $nssdir/cmd/manifest.mn $dstdir/cmd + cp $nssdir/cmd/platlibs.mk $dstdir/cmd + cp $nssdir/cmd/platrules.mk $dstdir/cmd + + # Copy some files at the top and the util subdirectory recursively. + mkdir $dstdir/lib + cp $nssdir/lib/Makefile $dstdir/lib + cp $nssdir/lib/manifest.mn $dstdir/lib + cp -R $nssdir/lib/util $dstdir/lib/util +} + +split_softoken() { + nssdir="$1" + dstdir="$2" + + # Prepare a source tree only containing files to build nss-softoken: + # + # nss/dbm full directory + # nss/coreconf full directory + # nss top files only + # nss/lib top files only + # nss/lib/freebl full directory + # nss/lib/softoken full directory + # nss/lib/softoken/dbm full directory + + # Copy everything. + cp -R $nssdir $dstdir + + # Skip gtests when building. + sed '/^DIRS = /s/ gtests$//' $nssdir/manifest.mn > $dstdir/manifest.mn-t && mv $dstdir/manifest.mn-t $dstdir/manifest.mn + + # Remove subdirectories that we don't want. + rm -rf $dstdir/cmd + rm -rf $dstdir/tests + rm -rf $dstdir/lib + rm -rf $dstdir/pkg + rm -rf $dstdir/automation + rm -rf $dstdir/gtests + rm -rf $dstdir/doc + + # Start with an empty lib directory and copy only what we need. + mkdir $dstdir/lib + copy_top $nssdir/lib $dstdir/lib + cp -R $nssdir/lib/dbm $dstdir/lib/dbm + cp -R $nssdir/lib/freebl $dstdir/lib/freebl + cp -R $nssdir/lib/softoken $dstdir/lib/softoken + cp -R $nssdir/lib/sqlite $dstdir/lib/sqlite + + mkdir $dstdir/cmd + copy_top $nssdir/cmd $dstdir/cmd + cp -R $nssdir/cmd/bltest $dstdir/cmd/bltest + cp -R $nssdir/cmd/ecperf $dstdir/cmd/ecperf + cp -R $nssdir/cmd/fbectest $dstdir/cmd/fbectest + cp -R $nssdir/cmd/fipstest $dstdir/cmd/fipstest + cp -R $nssdir/cmd/lib $dstdir/cmd/lib + cp -R $nssdir/cmd/lowhashtest $dstdir/cmd/lowhashtest + cp -R $nssdir/cmd/shlibsign $dstdir/cmd/shlibsign + + mkdir $dstdir/tests + copy_top $nssdir/tests $dstdir/tests + + cp -R $nssdir/tests/cipher $dstdir/tests/cipher + cp -R $nssdir/tests/common $dstdir/tests/common + cp -R $nssdir/tests/ec $dstdir/tests/ec + cp -R $nssdir/tests/lowhash $dstdir/tests/lowhash + + cp $nssdir/lib/util/verref.h $dstdir/lib/freebl + cp $nssdir/lib/util/verref.h $dstdir/lib/softoken + cp $nssdir/lib/util/verref.h $dstdir/lib/softoken/legacydb +} + +split_nss() { + nssdir="$1" + dstdir="$2" + + # Prepare a source tree only containing files to build nss: + # + # nss/dbm full directory + # nss/coreconf full directory + # nss top files only + # nss/lib top files only + # nss/lib/freebl full directory + # nss/lib/softoken full directory + # nss/lib/softoken/dbm full directory + + # Copy everything. + cp -R $nssdir $dstdir + + # Remove subdirectories that we don't want. + rm -rf $dstdir/lib/freebl + rm -rf $dstdir/lib/softoken + rm -rf $dstdir/lib/util + rm -rf $dstdir/cmd/bltest + rm -rf $dstdir/cmd/fipstest + rm -rf $dstdir/cmd/rsaperf_low + + # Copy these headers until the upstream bug is accepted + # Upstream https://bugzilla.mozilla.org/show_bug.cgi?id=820207 + cp $nssdir/lib/softoken/lowkeyi.h $dstdir/cmd/rsaperf + cp $nssdir/lib/softoken/lowkeyti.h $dstdir/cmd/rsaperf + + # Copy verref.h which will be needed later during the build phase. + cp $nssdir/lib/util/verref.h $dstdir/lib/ckfw/builtins/verref.h + cp $nssdir/lib/util/verref.h $dstdir/lib/nss/verref.h + cp $nssdir/lib/util/verref.h $dstdir/lib/smime/verref.h + cp $nssdir/lib/util/verref.h $dstdir/lib/ssl/verref.h + cp $nssdir/lib/util/templates.c $dstdir/lib/nss/templates.c + + # FIXME: Skip util_gtest because it links with libnssutil.a. Note + # that we can't use libnssutil3.so instead, because util_gtest + # depends on internal symbols not exported from the shared library. + sed '/ util_gtest \\/d' $dstdir/gtests/manifest.mn > $dstdir/gtests/manifest.mn-t && mv $dstdir/gtests/manifest.mn-t $dstdir/gtests/manifest.mn +} diff --git a/nss/automation/taskcluster/scripts/tools.sh b/nss/automation/taskcluster/scripts/tools.sh index dacfdeb..46d567e 100644 --- a/nss/automation/taskcluster/scripts/tools.sh +++ b/nss/automation/taskcluster/scripts/tools.sh @@ -2,11 +2,21 @@ set -v -e -x +if [[ $(id -u) -eq 0 ]]; then + # Drop privileges by re-running this script. + # Note: this mangles arguments, better to avoid running scripts as root. + exec su worker -c "$0 $*" +fi + # Usage: hg_clone repo dir [revision=@] hg_clone() { repo=$1 dir=$2 rev=${3:-@} + if [ -d "$dir" ]; then + hg pull -R "$dir" -ur "$rev" "$repo" && return + rm -rf "$dir" + fi for i in 0 2 5; do sleep $i hg clone -r "$rev" "$repo" "$dir" && return diff --git a/nss/automation/taskcluster/windows/build.sh b/nss/automation/taskcluster/windows/build.sh index 6c8a474..4613615 100644 --- a/nss/automation/taskcluster/windows/build.sh +++ b/nss/automation/taskcluster/windows/build.sh @@ -3,7 +3,11 @@ set -v -e -x # Set up the toolchain. -source $(dirname $0)/setup.sh +if [ "$USE_64" = 1 ]; then + source $(dirname $0)/setup64.sh +else + source $(dirname $0)/setup32.sh +fi # Clone NSPR. hg_clone https://hg.mozilla.org/projects/nspr nspr default diff --git a/nss/automation/taskcluster/windows/build_gyp.sh b/nss/automation/taskcluster/windows/build_gyp.sh new file mode 100644 index 0000000..cc829ca --- /dev/null +++ b/nss/automation/taskcluster/windows/build_gyp.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +set -v -e -x + +# Set up the toolchain. +if [[ "$@" == *"-m32"* ]]; then + source $(dirname $0)/setup32.sh +else + source $(dirname $0)/setup64.sh +fi + +# Install GYP. +cd gyp +python -m virtualenv test-env +test-env/Scripts/python setup.py install +test-env/Scripts/python -m pip install --upgrade pip +test-env/Scripts/pip install --upgrade setuptools +cd .. + +export GYP_MSVS_OVERRIDE_PATH="${VSPATH}" +export GYP_MSVS_VERSION="2015" +export GYP="${PWD}/gyp/test-env/Scripts/gyp" + +# Fool GYP. +touch "${VSPATH}/VC/vcvarsall.bat" + +# Clone NSPR. +hg_clone https://hg.mozilla.org/projects/nspr nspr default + +# Build with gyp. +GYP=${GYP} ./nss/build.sh -g -v "$@" + +# Package. +7z a public/build/dist.7z dist diff --git a/nss/automation/taskcluster/windows/releng.manifest b/nss/automation/taskcluster/windows/releng.manifest index b3f4498..68d2c1d 100644 --- a/nss/automation/taskcluster/windows/releng.manifest +++ b/nss/automation/taskcluster/windows/releng.manifest @@ -1,10 +1,26 @@ [ { - "version": "Visual Studio 2015 Update 2 / SDK 10.0.10586.0/212", - "size": 332442800, - "digest": "995394a4a515c7cb0f8595f26f5395361a638870dd0bbfcc22193fe1d98a0c47126057d5999cc494f3f3eac5cb49160e79757c468f83ee5797298e286ef6252c", + "version": "Visual Studio 2015 Update 3 14.0.25425.01 / SDK 10.0.14393.0", + "size": 326656969, + "digest": "babc414ffc0457d27f5a1ed24a8e4873afbe2f1c1a4075469a27c005e1babc3b2a788f643f825efedff95b79686664c67ec4340ed535487168a3482e68559bc7", "algorithm": "sha512", - "filename": "vs2015u2.zip", + "filename": "vs2015u3.zip", + "unpack": true + }, + { + "version": "Ninja 1.7.1", + "size": 184821, + "digest": "e4f9a1ae624a2630e75264ba37d396d9c7407d6e6aea3763056210ba6e1387908bd31cf4037a6a3661a418e86c4d2761e0c333e6a3bd0d66549d2b0d72d3f43b", + "algorithm": "sha512", + "filename": "ninja171.zip", + "unpack": true + }, + { + "size": 13063963, + "visibility": "public", + "digest": "47a19f8f863eab3414abab2b9e9bd901ab896c799b3d9254b456b2f59374b085b99de805e21069a0819f01eecb3f43f7e2395a8c644c04bcbfa5711261cca29d", + "algorithm": "sha512", + "filename": "gyp-2017-05-23.zip", "unpack": true } ] diff --git a/nss/automation/taskcluster/windows/setup.sh b/nss/automation/taskcluster/windows/setup.sh index 80cee28..7def50d 100644 --- a/nss/automation/taskcluster/windows/setup.sh +++ b/nss/automation/taskcluster/windows/setup.sh @@ -2,6 +2,13 @@ set -v -e -x +export VSPATH="$(pwd)/vs2015u3" +export NINJA_PATH="$(pwd)/ninja/bin" + +export WINDOWSSDKDIR="${VSPATH}/SDK" +export VS90COMNTOOLS="${VSPATH}/VC" +export INCLUDE="${VSPATH}/VC/include:${VSPATH}/SDK/Include/10.0.14393.0/ucrt:${VSPATH}/SDK/Include/10.0.14393.0/shared:${VSPATH}/SDK/Include/10.0.14393.0/um" + # Usage: hg_clone repo dir [revision=@] hg_clone() { repo=$1 @@ -16,15 +23,4 @@ hg_clone() { } hg_clone https://hg.mozilla.org/build/tools tools default - tools/scripts/tooltool/tooltool_wrapper.sh $(dirname $0)/releng.manifest https://api.pub.build.mozilla.org/tooltool/ non-existant-file.sh /c/mozilla-build/python/python.exe /c/builds/tooltool.py --authentication-file /c/builds/relengapi.tok -c /c/builds/tooltool_cache -VSPATH="$(pwd)/vs2015u2" - -export WINDOWSSDKDIR="${VSPATH}/SDK" -export WIN32_REDIST_DIR="${VSPATH}/VC/redist/x64/Microsoft.VC140.CRT" -export WIN_UCRT_REDIST_DIR="${VSPATH}/SDK/Redist/ucrt/DLLs/x64" - -export PATH="${VSPATH}/VC/bin/amd64:${VSPATH}/VC/bin:${VSPATH}/SDK/bin/x64:${VSPATH}/VC/redist/x64/Microsoft.VC140.CRT:${VSPATH}/SDK/Redist/ucrt/DLLs/x64:${PATH}" - -export INCLUDE="${VSPATH}/VC/include:${VSPATH}/SDK/Include/10.0.10586.0/ucrt:${VSPATH}/SDK/Include/10.0.10586.0/shared:${VSPATH}/SDK/Include/10.0.10586.0/um" -export LIB="${VSPATH}/VC/lib/amd64:${VSPATH}/SDK/lib/10.0.10586.0/ucrt/x64:${VSPATH}/SDK/lib/10.0.10586.0/um/x64" diff --git a/nss/automation/taskcluster/windows/setup32.sh b/nss/automation/taskcluster/windows/setup32.sh new file mode 100644 index 0000000..bcddabf --- /dev/null +++ b/nss/automation/taskcluster/windows/setup32.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -v -e -x + +source $(dirname $0)/setup.sh + +export WIN32_REDIST_DIR="${VSPATH}/VC/redist/x86/Microsoft.VC140.CRT" +export WIN_UCRT_REDIST_DIR="${VSPATH}/SDK/Redist/ucrt/DLLs/x86" +export PATH="${NINJA_PATH}:${VSPATH}/VC/bin/amd64_x86:${VSPATH}/VC/bin/amd64:${VSPATH}/VC/bin:${VSPATH}/SDK/bin/x86:${VSPATH}/SDK/bin/x64:${VSPATH}/VC/redist/x86/Microsoft.VC140.CRT:${VSPATH}/VC/redist/x64/Microsoft.VC140.CRT:${VSPATH}/SDK/Redist/ucrt/DLLs/x86:${VSPATH}/SDK/Redist/ucrt/DLLs/x64:${PATH}" +export LIB="${VSPATH}/VC/lib:${VSPATH}/SDK/lib/10.0.14393.0/ucrt/x86:${VSPATH}/SDK/lib/10.0.14393.0/um/x86" diff --git a/nss/automation/taskcluster/windows/setup64.sh b/nss/automation/taskcluster/windows/setup64.sh new file mode 100644 index 0000000..f308298 --- /dev/null +++ b/nss/automation/taskcluster/windows/setup64.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -v -e -x + +source $(dirname $0)/setup.sh + +export WIN32_REDIST_DIR="${VSPATH}/VC/redist/x64/Microsoft.VC140.CRT" +export WIN_UCRT_REDIST_DIR="${VSPATH}/SDK/Redist/ucrt/DLLs/x64" +export PATH="${NINJA_PATH}:${VSPATH}/VC/bin/amd64:${VSPATH}/VC/bin:${VSPATH}/SDK/bin/x64:${VSPATH}/VC/redist/x64/Microsoft.VC140.CRT:${VSPATH}/SDK/Redist/ucrt/DLLs/x64:${PATH}" +export LIB="${VSPATH}/VC/lib/amd64:${VSPATH}/SDK/lib/10.0.14393.0/ucrt/x64:${VSPATH}/SDK/lib/10.0.14393.0/um/x64" |