From 76a7d5f1728745889a0d53c8dc498cbc5eb6dbdb Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Wed, 26 Apr 2017 13:04:23 +0200 Subject: travis: cibuild: set up our own sshd server Some tests of ours require to be running against an SSH server. Currently, we simply run against the SSH server provided and started by Travis itself. As our Linux tests run in a sudo-less environment, we have no control over its configuration and startup/shutdown procedure. While this has been no problem until now, it will become a problem as soon as we migrate over to newer Precise images, as the SSH server does not have any host keys set up. Luckily, we can simply set up our own unpriviledged SSH server. This has the benefit of us being able to modify its configuration even in a sudo-less environment. This commit sets up the unpriviledged SSH server on port 2222. (cherry picked from commit 06619904a2ae2ffd5d8e34ab11d5eb484e9d5762) --- script/cibuild.sh | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) (limited to 'script/cibuild.sh') diff --git a/script/cibuild.sh b/script/cibuild.sh index 403df223e..9239136dd 100755 --- a/script/cibuild.sh +++ b/script/cibuild.sh @@ -43,23 +43,39 @@ ctest -V -R libgit2_clar || exit $? killall git-daemon -if [ "$TRAVIS_OS_NAME" = "osx" ]; then - echo 'PasswordAuthentication yes' | sudo tee -a /etc/sshd_config -fi - +# Set up sshd +mkdir ~/sshd/ +cat >~/sshd/sshd_config<<-EOF + Port 2222 + ListenAddress 0.0.0.0 + Protocol 2 + HostKey ${HOME}/sshd/id_rsa + RSAAuthentication yes + PasswordAuthentication yes + PubkeyAuthentication yes + ChallengeResponseAuthentication no + # Required here as sshd will simply close connection otherwise + UsePAM no +EOF +ssh-keygen -t rsa -f ~/sshd/id_rsa -N "" -q +/usr/sbin/sshd -f ~/sshd/sshd_config + +# Set up keys ssh-keygen -t rsa -f ~/.ssh/id_rsa -N "" -q cat ~/.ssh/id_rsa.pub >>~/.ssh/authorized_keys -ssh-keyscan -t rsa localhost >>~/.ssh/known_hosts +while read algorithm key comment; do + echo "[localhost]:2222 $algorithm $key" >>~/.ssh/known_hosts +done <~/sshd/id_rsa.pub # Get the fingerprint for localhost and remove the colons so we can parse it as # a hex number. The Mac version is newer so it has a different output format. if [ "$TRAVIS_OS_NAME" = "osx" ]; then - export GITTEST_REMOTE_SSH_FINGERPRINT=$(ssh-keygen -E md5 -F localhost -l | tail -n 1 | cut -d ' ' -f 3 | cut -d : -f2- | tr -d :) + export GITTEST_REMOTE_SSH_FINGERPRINT=$(ssh-keygen -E md5 -F '[localhost]:2222' -l | tail -n 1 | cut -d ' ' -f 3 | cut -d : -f2- | tr -d :) else - export GITTEST_REMOTE_SSH_FINGERPRINT=$(ssh-keygen -F localhost -l | tail -n 1 | cut -d ' ' -f 2 | tr -d ':') + export GITTEST_REMOTE_SSH_FINGERPRINT=$(ssh-keygen -F '[localhost]:2222' -l | tail -n 1 | cut -d ' ' -f 2 | tr -d ':') fi -export GITTEST_REMOTE_URL="ssh://localhost/$HOME/_temp/test.git" +export GITTEST_REMOTE_URL="ssh://localhost:2222/$HOME/_temp/test.git" export GITTEST_REMOTE_USER=$USER export GITTEST_REMOTE_SSH_KEY="$HOME/.ssh/id_rsa" export GITTEST_REMOTE_SSH_PUBKEY="$HOME/.ssh/id_rsa.pub" @@ -83,6 +99,8 @@ if [ -e ./libgit2_clar ]; then fi +killall sshd + export GITTEST_REMOTE_URL="https://github.com/libgit2/non-existent" export GITTEST_REMOTE_USER="libgit2test" ctest -V -R libgit2_clar-cred_callback -- cgit v1.2.1 From 7d1c72a4688418095c04324b59685c4a79c5596a Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Mon, 24 Jul 2017 16:48:04 +0100 Subject: travis: only kill our own sshd (cherry picked from commit 697583ea3aceb1379c576515ffa713ba29c50437) --- script/cibuild.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'script/cibuild.sh') diff --git a/script/cibuild.sh b/script/cibuild.sh index 9239136dd..74946db0a 100755 --- a/script/cibuild.sh +++ b/script/cibuild.sh @@ -50,6 +50,7 @@ cat >~/sshd/sshd_config<<-EOF ListenAddress 0.0.0.0 Protocol 2 HostKey ${HOME}/sshd/id_rsa + PidFile ${HOME}/sshd/pid RSAAuthentication yes PasswordAuthentication yes PubkeyAuthentication yes @@ -99,7 +100,7 @@ if [ -e ./libgit2_clar ]; then fi -killall sshd +kill $(cat "$HOME/sshd/pid") export GITTEST_REMOTE_URL="https://github.com/libgit2/non-existent" export GITTEST_REMOTE_USER="libgit2test" -- cgit v1.2.1 From 4eecbdd0be15fc497892f970a41d959402546a7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= Date: Tue, 31 Oct 2017 10:40:24 +0100 Subject: travis: put clar's sandbox in a ramdisk on macOS The macOS tests are by far the slowest right now. This attempts to remedy the situation somewhat by asking clar to put its test data on a ramdisk. (cherry picked from commit 37bb15122e30bb13aabc213079da53b5cdac2678) --- script/cibuild.sh | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'script/cibuild.sh') diff --git a/script/cibuild.sh b/script/cibuild.sh index 74946db0a..283acbf5b 100755 --- a/script/cibuild.sh +++ b/script/cibuild.sh @@ -10,6 +10,15 @@ fi if [ "$TRAVIS_OS_NAME" = "osx" ]; then export PKG_CONFIG_PATH=$(ls -d /usr/local/Cellar/{curl,zlib}/*/lib/pkgconfig | paste -s -d':' -) + + # Set up a ramdisk for us to put our test data on to speed up tests on macOS + export CLAR_TMP="$HOME"/_clar_tmp + mkdir -p $CLAR_TMP + + # 2M sectors aka ~1GB of space + device=$(hdiutil attach -nomount ram://$((2 * 1024 * 1024))) + newfs_hfs $device + mount -t hfs $device $CLAR_TMP fi # Should we ask Travis to cache this file? -- cgit v1.2.1 From 9343482873b512183209b33d356f7fb8f6297291 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= Date: Tue, 31 Oct 2017 14:43:28 +0100 Subject: travis: let's try a 5GB ramdisk (cherry picked from commit 71ba464435bb430b02d94c653cd518c11f7289ff) --- script/cibuild.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'script/cibuild.sh') diff --git a/script/cibuild.sh b/script/cibuild.sh index 283acbf5b..1c28baae6 100755 --- a/script/cibuild.sh +++ b/script/cibuild.sh @@ -15,8 +15,8 @@ if [ "$TRAVIS_OS_NAME" = "osx" ]; then export CLAR_TMP="$HOME"/_clar_tmp mkdir -p $CLAR_TMP - # 2M sectors aka ~1GB of space - device=$(hdiutil attach -nomount ram://$((2 * 1024 * 1024))) + # 5*2M sectors aka ~5GB of space + device=$(hdiutil attach -nomount ram://$((5 * 2 * 1024 * 1024))) newfs_hfs $device mount -t hfs $device $CLAR_TMP fi -- cgit v1.2.1 From a1a495f21e58366058ea17a6fe8cab8aa4ad2359 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Wed, 7 Jun 2017 12:48:48 +0200 Subject: tests: online::clone: construct credential-URL from environment We support two types of passing credentials to the proxy, either via the URL or explicitly by specifying user and password. We test these types by modifying the proxy URL and executing the tests twice, which is in fact unnecessary and requires us to maintain the list of environment variables and test executions across multiple CI infrastructures. To fix the situation, we can just always pass the host, port, user and password to the tests. The tests can then assemble the complete URL either with or without included credentials, allowing us to test both cases in-process. (cherry picked from commit fea6092079d5c09b499e472efead2f7aa81ce8a1) --- script/cibuild.sh | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) (limited to 'script/cibuild.sh') diff --git a/script/cibuild.sh b/script/cibuild.sh index 1c28baae6..c06de1933 100755 --- a/script/cibuild.sh +++ b/script/cibuild.sh @@ -90,7 +90,10 @@ export GITTEST_REMOTE_USER=$USER export GITTEST_REMOTE_SSH_KEY="$HOME/.ssh/id_rsa" export GITTEST_REMOTE_SSH_PUBKEY="$HOME/.ssh/id_rsa.pub" export GITTEST_REMOTE_SSH_PASSPHRASE="" - +# Use the proxy we started at the beginning +export GITTEST_REMOTE_PROXY_URL="localhost:8080" +export GITTEST_REMOTE_PROXY_USER="foo" +export GITTEST_REMOTE_PROXY_PASS="bar" if [ -e ./libgit2_clar ]; then ./libgit2_clar -sonline::push -sonline::clone::ssh_cert && @@ -99,14 +102,7 @@ if [ -e ./libgit2_clar ]; then ./libgit2_clar -sonline::clone::cred_callback || exit $? fi - # Use the proxy we started at the beginning - export GITTEST_REMOTE_PROXY_URL="http://foo:bar@localhost:8080/" - ./libgit2_clar -sonline::clone::proxy_credentials_in_url || exit $? - export GITTEST_REMOTE_PROXY_URL="http://localhost:8080/" - export GITTEST_REMOTE_PROXY_USER="foo" - export GITTEST_REMOTE_PROXY_PASS="bar" - ./libgit2_clar -sonline::clone::proxy_credentials_request || exit $? - + ctest -V -R libgit2_clar-proxy_credentials || exit $? fi kill $(cat "$HOME/sshd/pid") -- cgit v1.2.1 From 2362ce6ce9bf87d60fe888c96c73e6a4a66c3de1 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Wed, 7 Jun 2017 13:06:53 +0200 Subject: tests: online::clone: inline creds-test with nonexistent URL Right now, we test our credential callback code twice, once via SSH on localhost and once via a non-existent GitHub repository. While the first URL makes sense to be configurable, it does not make sense to hard-code the non-existing repository, which requires us to call tests multiple times. Instead, we can just inline the URL into another set of tests. (cherry picked from commit 54a1bf057a1123cf55ac3447c79761c817382f47) --- script/cibuild.sh | 7 ------- 1 file changed, 7 deletions(-) (limited to 'script/cibuild.sh') diff --git a/script/cibuild.sh b/script/cibuild.sh index c06de1933..5cebb5c17 100755 --- a/script/cibuild.sh +++ b/script/cibuild.sh @@ -98,15 +98,8 @@ export GITTEST_REMOTE_PROXY_PASS="bar" if [ -e ./libgit2_clar ]; then ./libgit2_clar -sonline::push -sonline::clone::ssh_cert && ./libgit2_clar -sonline::clone::ssh_with_paths || exit $? - if [ "$TRAVIS_OS_NAME" = "linux" ]; then - ./libgit2_clar -sonline::clone::cred_callback || exit $? - fi ctest -V -R libgit2_clar-proxy_credentials || exit $? fi kill $(cat "$HOME/sshd/pid") - -export GITTEST_REMOTE_URL="https://github.com/libgit2/non-existent" -export GITTEST_REMOTE_USER="libgit2test" -ctest -V -R libgit2_clar-cred_callback -- cgit v1.2.1 From 637412cc475bdcade43b690779ccb260c0d12a9c Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Mon, 20 Nov 2017 13:26:33 +0000 Subject: tests: create new test target for all SSH-based tests Some tests shall be run against our own SSH server we spin up in Travis. As those need to be run separate from our previous tests which run against git-daemon, we have to do this in a separate step. Instead of bundling all that knowledge in the CI script, move it into the test build instructions by creating a new test target. (cherry picked from commit 5874e151d7b10de84fc1ca168339fdc622292219) --- script/cibuild.sh | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) (limited to 'script/cibuild.sh') diff --git a/script/cibuild.sh b/script/cibuild.sh index 5cebb5c17..5d70e7506 100755 --- a/script/cibuild.sh +++ b/script/cibuild.sh @@ -85,21 +85,18 @@ else export GITTEST_REMOTE_SSH_FINGERPRINT=$(ssh-keygen -F '[localhost]:2222' -l | tail -n 1 | cut -d ' ' -f 2 | tr -d ':') fi +# Use the SSH server export GITTEST_REMOTE_URL="ssh://localhost:2222/$HOME/_temp/test.git" export GITTEST_REMOTE_USER=$USER export GITTEST_REMOTE_SSH_KEY="$HOME/.ssh/id_rsa" export GITTEST_REMOTE_SSH_PUBKEY="$HOME/.ssh/id_rsa.pub" export GITTEST_REMOTE_SSH_PASSPHRASE="" +ctest -V -R libgit2_clar-ssh || exit $? + # Use the proxy we started at the beginning export GITTEST_REMOTE_PROXY_URL="localhost:8080" export GITTEST_REMOTE_PROXY_USER="foo" export GITTEST_REMOTE_PROXY_PASS="bar" - -if [ -e ./libgit2_clar ]; then - ./libgit2_clar -sonline::push -sonline::clone::ssh_cert && - ./libgit2_clar -sonline::clone::ssh_with_paths || exit $? - - ctest -V -R libgit2_clar-proxy_credentials || exit $? -fi +ctest -V -R libgit2_clar-proxy_credentials || exit $? kill $(cat "$HOME/sshd/pid") -- cgit v1.2.1 From e42f8f7363bf06f437608dcf4c277f8123d57c34 Mon Sep 17 00:00:00 2001 From: Etienne Samson Date: Fri, 20 Apr 2018 23:11:14 +0200 Subject: travis: split testing from building (cherry picked from commit 2f4e7cb0e8c21cc2d673946eddf9278c2863427b) --- script/cibuild.sh | 74 ------------------------------------------------------- 1 file changed, 74 deletions(-) (limited to 'script/cibuild.sh') diff --git a/script/cibuild.sh b/script/cibuild.sh index 5d70e7506..42705b154 100755 --- a/script/cibuild.sh +++ b/script/cibuild.sh @@ -21,82 +21,8 @@ if [ "$TRAVIS_OS_NAME" = "osx" ]; then mount -t hfs $device $CLAR_TMP fi -# Should we ask Travis to cache this file? -curl -L https://github.com/ethomson/poxyproxy/releases/download/v0.1.0/poxyproxy-0.1.0.jar >poxyproxy.jar || exit $? -# Run this early so we know it's ready by the time we need it -java -jar poxyproxy.jar -d --port 8080 --credentials foo:bar & - mkdir _build cd _build # shellcheck disable=SC2086 cmake .. -DBUILD_EXAMPLES=ON -DCMAKE_INSTALL_PREFIX=../_install $OPTIONS || exit $? make -j2 install || exit $? - -# If this platform doesn't support test execution, bail out now -if [ -n "$SKIP_TESTS" ]; -then - exit $?; -fi - -# Create a test repo which we can use for the online::push tests -mkdir "$HOME"/_temp -git init --bare "$HOME"/_temp/test.git -git daemon --listen=localhost --export-all --enable=receive-pack --base-path="$HOME"/_temp "$HOME"/_temp 2>/dev/null & -export GITTEST_REMOTE_URL="git://localhost/test.git" - -# Run the test suite -ctest -V -R libgit2_clar || exit $? - -# Now that we've tested the raw git protocol, let's set up ssh to we -# can do the push tests over it - -killall git-daemon - -# Set up sshd -mkdir ~/sshd/ -cat >~/sshd/sshd_config<<-EOF - Port 2222 - ListenAddress 0.0.0.0 - Protocol 2 - HostKey ${HOME}/sshd/id_rsa - PidFile ${HOME}/sshd/pid - RSAAuthentication yes - PasswordAuthentication yes - PubkeyAuthentication yes - ChallengeResponseAuthentication no - # Required here as sshd will simply close connection otherwise - UsePAM no -EOF -ssh-keygen -t rsa -f ~/sshd/id_rsa -N "" -q -/usr/sbin/sshd -f ~/sshd/sshd_config - -# Set up keys -ssh-keygen -t rsa -f ~/.ssh/id_rsa -N "" -q -cat ~/.ssh/id_rsa.pub >>~/.ssh/authorized_keys -while read algorithm key comment; do - echo "[localhost]:2222 $algorithm $key" >>~/.ssh/known_hosts -done <~/sshd/id_rsa.pub - -# Get the fingerprint for localhost and remove the colons so we can parse it as -# a hex number. The Mac version is newer so it has a different output format. -if [ "$TRAVIS_OS_NAME" = "osx" ]; then - export GITTEST_REMOTE_SSH_FINGERPRINT=$(ssh-keygen -E md5 -F '[localhost]:2222' -l | tail -n 1 | cut -d ' ' -f 3 | cut -d : -f2- | tr -d :) -else - export GITTEST_REMOTE_SSH_FINGERPRINT=$(ssh-keygen -F '[localhost]:2222' -l | tail -n 1 | cut -d ' ' -f 2 | tr -d ':') -fi - -# Use the SSH server -export GITTEST_REMOTE_URL="ssh://localhost:2222/$HOME/_temp/test.git" -export GITTEST_REMOTE_USER=$USER -export GITTEST_REMOTE_SSH_KEY="$HOME/.ssh/id_rsa" -export GITTEST_REMOTE_SSH_PUBKEY="$HOME/.ssh/id_rsa.pub" -export GITTEST_REMOTE_SSH_PASSPHRASE="" -ctest -V -R libgit2_clar-ssh || exit $? - -# Use the proxy we started at the beginning -export GITTEST_REMOTE_PROXY_URL="localhost:8080" -export GITTEST_REMOTE_PROXY_USER="foo" -export GITTEST_REMOTE_PROXY_PASS="bar" -ctest -V -R libgit2_clar-proxy_credentials || exit $? - -kill $(cat "$HOME/sshd/pid") -- cgit v1.2.1 From 4e7bff0d1fc6d34c0ecb2616a80ba96f541fa480 Mon Sep 17 00:00:00 2001 From: Etienne Samson Date: Fri, 20 Apr 2018 23:11:23 +0200 Subject: travis: let cmake perform the build & install step The goal is to let cmake manage the parallelism (cherry picked from commit 1f4ada2a428c8d4af3cc0f12086700cda6e19e3a) --- script/cibuild.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'script/cibuild.sh') diff --git a/script/cibuild.sh b/script/cibuild.sh index 42705b154..43ca976cc 100755 --- a/script/cibuild.sh +++ b/script/cibuild.sh @@ -25,4 +25,4 @@ mkdir _build cd _build # shellcheck disable=SC2086 cmake .. -DBUILD_EXAMPLES=ON -DCMAKE_INSTALL_PREFIX=../_install $OPTIONS || exit $? -make -j2 install || exit $? +cmake --build . --target install || exit $? -- cgit v1.2.1 From 590d46d6870039195d8becd13169ad016569de6e Mon Sep 17 00:00:00 2001 From: Etienne Samson Date: Fri, 20 Apr 2018 23:11:28 +0200 Subject: scripts: remove extraneous semicolons (cherry picked from commit 149790b96eda8a1e48408decf92ba327479c2c33) --- script/cibuild.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'script/cibuild.sh') diff --git a/script/cibuild.sh b/script/cibuild.sh index 43ca976cc..c062c112e 100755 --- a/script/cibuild.sh +++ b/script/cibuild.sh @@ -2,10 +2,9 @@ set -x -if [ -n "$COVERITY" ]; -then - ./script/coverity.sh; - exit $?; +if [ -n "$COVERITY" ]; then + ./script/coverity.sh + exit $? fi if [ "$TRAVIS_OS_NAME" = "osx" ]; then -- cgit v1.2.1