From: Ward Vandewege Date: Tue, 28 Jul 2020 17:50:12 +0000 (-0400) Subject: 16616: move jenkins packer templates here. X-Git-Url: https://git.arvados.org/arvados-dev.git/commitdiff_plain/971115bd0eebb58de860a29a4a3d3a9985a3f719 16616: move jenkins packer templates here. Arvados-DCO-1.1-Signed-off-by: Ward Vandewege --- diff --git a/.licenseignore b/.licenseignore new file mode 100644 index 0000000..5374b2c --- /dev/null +++ b/.licenseignore @@ -0,0 +1,6 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 +jenkins/packer-images/.gitignore +jenkins/packer-images/*.json +jenkins/packer-images/1078ECD7.asc diff --git a/jenkins/packer-images/.gitignore b/jenkins/packer-images/.gitignore new file mode 100644 index 0000000..634bc50 --- /dev/null +++ b/jenkins/packer-images/.gitignore @@ -0,0 +1 @@ +gce_googlecompute.pem diff --git a/jenkins/packer-images/1078ECD7.asc b/jenkins/packer-images/1078ECD7.asc new file mode 100644 index 0000000..edc62f4 --- /dev/null +++ b/jenkins/packer-images/1078ECD7.asc @@ -0,0 +1,30 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQENBEzhgeoBCAChhoK1dqpWzNyDWqRGEvdFdkJaA9D2HRwKPfBfjAoePX6ZyrpA +ItlUsvt/8s/DRiTiPEFQR4S7VqocmU6whJc3gDEGyOM6b1NF873lIfSVwUoE42QE +a76dO8woOYgLUyxu2mKG+bJgGMumjBJt6ZOndYVjTYB/7sEeVxwmMVulfZe0s6zg +ut0+SoTYg2R36qIqeIcWllYt97sEYnyy1qXMis4/3IZnuWkS/frsPR3aeUI4W+o2 +NDN1kj49+LMe7Fb5b7jZY08rZbAWXi1rU1hQx4jC9RvYqlT4HNld4Bn7os1IvOOA +wNiR0oiVdiuDbBxcMvRPktxMrFVjowusRLq/ABEBAAG0PUN1cm92ZXJzZSwgSW5j +IEF1dG9tYXRpYyBTaWduaW5nIEtleSA8c3lzYWRtaW5AY3Vyb3ZlcnNlLmNvbT6J +ATgEEwECACIFAlNgYIECGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEFcW +WREQeOzXPkEH/jQJDIYI1dxWcYiA+hczmpaZvN2/pc/kwIW/6a03+6zqmSNkebOE +TgoDILacSYc17hy20R1/rWyUstOMKcEgFDBlSehhHyl0f7q/w7d8Ais6MabzsPfx +IceJpsjUg87+BR7qWhgQ0sxmtIF2TKuTFLs+nkGsgSsiBOEF4NvHxuj3HD4y8F27 +HNqrkqwjLS8xJwwH5Gp2uMEVr1AXIH3iSRjJ8X124s8iEP97Q/3IazoYRf9/MCSm +QEx8KzxwDX6t4bW6O4D01K+e9gdkTY70dcMgJoqm5IsX7yxjEubiOunphtlJnZ9d +Oi1yBN5UM3pWKAdcfRj4rcfV9Simvpx9av+5AQ0ETOGB6gEIAMAA0HVMG0BbdnU7 +wWgl5eFdT0AUSrXK/WdcKqVEGGv+c68NETSHWZOJX7O46Eao4gY4cTYprVMBzxpY +/BtQSYLpE0HLvBc1fcFd61Yz4H/9rGSNY0GcIQEbOjbJY5mr8qFsQ1K/mAf3aUL3 +b6ni4sHVicRiRr0Gl4Ihorlskpfu1SHs/C5tvTSVNF9p4vtl5892y1yILQeVpcBs +NCR7MUpdS49xCpvnAWsDZX+ij6LTR3lzCm/ZLCg4gNuZkjgU9oqVfGkqysW7WZ8S +OLvzAwUw7i1EIFX8q6QdudGoezxz8m8OgZM1v8AFpYEKlhEPf1W0MSfaRDwrj866 +8nCLruEAEQEAAYkBHwQYAQIACQUCTOGB6gIbDAAKCRBXFlkREHjs199EB/4+p0G1 +3PHxt6rLWSCGXobDOu4ZOA/qnv0D/JhOLroFds5TzQv6vnS8eAkhCTjHVA+b58cm +kXpI0oYcD4ZP+KK1CHKq2rGfwou7HfAF+icnNqYkeBOkjjbCgkvBlcCInuAuU8JX +DZMkfFk52+eBKwTjS/J/fQp0vDru8bHLp98WgdRHWfJQ3mc3gz4A5sR6zhrGPW6/ +ssnROS4dC2Ohp35GpgN1KjD3EmEw5RoSBYlyrARCaMsivgIKMxGUEyFZWhuJt3N1 +2MTddRwz28hbmYCi+MzHYDbRv+cSyUDmvXaWhfkNKBepClBA1rTWBcldit5vvlqr +yPet6wIKrtLGhAqZ +=CLkG +-----END PGP PUBLIC KEY BLOCK----- diff --git a/jenkins/packer-images/README b/jenkins/packer-images/README new file mode 100644 index 0000000..cbd71b4 --- /dev/null +++ b/jenkins/packer-images/README @@ -0,0 +1,22 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: CC-BY-SA-3.0 + +The files under this directory are used to create images used in our Jenkins CI. + +*** IMAGE FOR RUNNING TESTS *** + +packer build jenkins-image-tests.json + +*** IMAGE FOR RUNNING FEDERATION TESTS *** + +packer build jenkins-image-federation-tests.json + +*** IMAGE FOR BUILDING PACKAGES *** + +packer build jenkins-image-build-packages.json + +*** IMAGE FOR K8S TESTS *** + +packer build jenkins-image-k8s-tests.json + diff --git a/jenkins/packer-images/jenkins-image-build-packages.json b/jenkins/packer-images/jenkins-image-build-packages.json new file mode 100644 index 0000000..b14d285 --- /dev/null +++ b/jenkins/packer-images/jenkins-image-build-packages.json @@ -0,0 +1,33 @@ +{ + "variables": { + "key_file": "{{env `packerbuildkey`}}" + }, + "builders": [ + { + "type": "googlecompute", + "account_file": "{{user `key_file` }}", + "project_id": "phoenix-project-42-ci", + "source_image": "debian-10-buster-v20200413", + "zone": "us-central1-b", + "disk_size": "20", + "image_name": "build-packages-jenkins-image-{{isotime \"20060102150405\"}}", + "image_family": "jenkins", + "ssh_username": "jenkins" + } + ], + "provisioners": [ + { + "type": "file", + "source": "node-ready.sh", + "destination": "/tmp/node-ready.sh" + }, + { + "type": "shell", + "script": "./jenkins-image-common.sh" + }, + { + "type": "shell", + "script": "./jenkins-image-build-packages.sh" + } + ] +} diff --git a/jenkins/packer-images/jenkins-image-build-packages.sh b/jenkins/packer-images/jenkins-image-build-packages.sh new file mode 100755 index 0000000..1922964 --- /dev/null +++ b/jenkins/packer-images/jenkins-image-build-packages.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +set -eo pipefail + +# Install the dependencies for the package building/testing jobs +sudo su -c "DEBIAN_FRONTEND=noninteractive apt-get install -y docker.io make wget dpkg-dev createrepo unzip" +sudo usermod -a -G docker jenkins + +#Packer install +cd /tmp +wget https://releases.hashicorp.com/packer/1.6.0/packer_1.6.0_linux_amd64.zip +unzip packer_1.6.0_linux_amd64.zip packer +sudo mv packer /usr/local/bin/ + +# Install the arvados-dev repo where the Jenkins job expects it +cd /usr/local +sudo git clone --depth 1 https://github.com/curoverse/arvados-dev +sudo chown -R jenkins:jenkins /usr/local/arvados-dev/ diff --git a/jenkins/packer-images/jenkins-image-common.sh b/jenkins/packer-images/jenkins-image-common.sh new file mode 100755 index 0000000..460e22c --- /dev/null +++ b/jenkins/packer-images/jenkins-image-common.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +set -eo pipefail + +sudo su -c "echo ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAgEA3nzI6T6Lpd5xFoRewcx91Dv9sUzNNmdYfwOleemBFz0y3RaQElehUWasyjuIURZw7RL5EjvrqeQq9pe/lO99dO0F9yMuMsMH2t88YrVJQ/z/5Aa4I2zYQotKb/9CCfynsy41y5xywxtOwXiDk2kpo+c9VZCyCeW8Hnc9HaIpkKSLnkqDVhESzlrkYyNKZvUAL1hiFIzmmw/veFgRb7/ol76Ze3xsWugbHUECEIAKoz/8uaevOAAoJrMFhffFIQ8IfClqDZv2lnBBhvh1O9TO0Mg4klcieyQ1RZhMjeP4WnAa9PXP7xZlQHLgO9qO1jDd2sOdkX6EedCfX5jO4Y51HPJpV35uYumw3veftMlpIJFmA2eIQxU19SCYpojWRGXZ5v9WtFIHX2nGy+Gi1bk7TR+HBsCDXOPhhQk4ceIM4OonqEb1NJ57elxh6mFbDAQCZtYhRLqYvcyGpuBdVdLNOJbOZ7vBJY3Kfjxa87rvFIJheT6DXhpdRayeOovLm0vuJ53bZoWxWOqjpuigQqHtSB3OmintrKB916BhNFsHwPeZpK0ahZGFygV63REM/X8m0nOlHqyAY69uzLHyYXM83zAI1L5Y4wuzsVqO+1tnK6PMcfg5/DArjBrn7YqA5tzJY9EgdVVPjpgjD2yYYTOP6b3UUw4uFj3asnOX24dfVzk= ward@countzero" >> /home/jenkins/.ssh/authorized_keys +sudo su -c "echo ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCfRJenfxGPFuJ/W2KUs6Wf0waiFsSaxd8WUTFXi4ophH2qkUMxd8xvoex/Z+Iq1fA8QCe4DfFnhUOMViP9bBB6+Va/x1Jx521P+DfN8ILqM2kcug0OoC4c5gTyFecJaKIyX6jMeidQThkd/UPHeTdicOnMqUTKIksQVXzIxzve6MQxbZts1viw0VgcO2t16F9LA+JaB8ONOTPfBzJ/BzZRiztfzQ0qW9rpX23icHtv494QrHZtgpvXltB/fy+YUZ80QJH1Iul4xnNL/27/MtZempJSSzvzlZxMIpRbe9F4qn859DZMGjUnoVMQ5v6Aw7fQRm1iMVa++i6i5X0bAI6xR6eIQzPgGdu+ymlanH0TO+5aOhOcuELyL81q2VBl01SBCfZ2OxX2ZL1HCLMe7vtxl8qxbO7Ur/Cq6j6/t7Ghb/jSnxY68rNm7MhOc5bcWL6vPxjmDMPzpk1bKL0lfEOoI4Dij+Rb0p8glC2399lZrMUTspf+/Qz6bP6ev/Z2ZJDeugRBQeLcMdu5dg1CzIKf+x8n3IbSyu5w/ML9BGq6vXpViVVkZ73DYo1BipnUVpzZ+wVGk+JdhoBFD3wlZYm/rDMiMtKrtcYhvPrvp3EhM+vBbCkcAKY40mcVuQD8BhPe5za9vlqjTKHOUTOGLkI4RAiPF3kwjXwELyxuOIki5w== javier@agnes RSA-4096, 20170323" >> /home/jenkins/.ssh/authorized_keys +sudo su -c "echo ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDH8swFWEfEfHhA+C5ezV8SXO/PkzGD1SH5VAQP/XDIrtnUocBZ3CE30lSyqYJI/EVKVqVa/ICQ0YpUwiMK6+3Jr9QQJwVyTmPji2nY3InL+1XAucN6HFJGKY9bYSsNOuKooj22GwBWw3gfJNLg/8qtpVykEq1yRpyh6pGsXT+J5nUZ723vZZTh//sxdN4CM8D8zoDgHc4RbL+zvESnCDrDbtMhg2u1h14RWiFOBAnzYuWcgtVDy2HA9iS0hJFB2UOV50byXLrEetxJ84PTwRsV2irq1y63g58VxwYOUrVZ08MY5qFvHExBjPqeqhRMzE7GufWM5F1CcUuGviOGFWfqMnfG4VOirPkFtRoK2oKRxH+NVPoUXWWxItJQ1dZ9hLDDWgAbxAvLS4Nnl2hvOVAbC7RVpXfoAhIPpL48oS1UprbsZIMxk2ZmRSJB1ykD3aLUvoO4zoD6xADt8uLiPvVYgFWUy1doLxHZqdY1Omc91owgQVPKvQ4vhqsJehQl4ZDS+O+8S7aC5m8sQ/V+NqiiXLH22vN58K7qNrkHWdb1n+rhilMbA5zp3cSKBgwmmNdupyPkJOKvf3IS7i4El+c8RFmRQv4FzGrdjGXAP8LPtt1dWPgHTFYjmrkOHLmfWM/y8cuyPWW/HEp3Y/msPQRlS3Gymce//vAWgN4T9yN46w== lucas@notebook" >> /home/jenkins/.ssh/authorized_keys + +echo "deb http://deb.debian.org/debian buster-backports main" | sudo tee /etc/apt/sources.list.d/buster-backports.list + +# Install a few dependency packages +sudo su -c "apt-get update" +sudo su -c "DEBIAN_FRONTEND=noninteractive apt install -y git netcat default-jdk" + +# create a reference repository (bare git repo) +# jenkins will use this to speed up the checkout for each job +cd /usr/src +sudo git clone --mirror https://git.arvados.org/arvados.git +sudo chown jenkins:jenkins arvados.git -R + +# Jenkins will use this script to determine when the node is ready for use +sudo mv /tmp/node-ready.sh /usr/local/bin/ + +# make sure sshd does not start on boot (yes, this is nasty). Jenkins will call +# /tmp/node-ready.sh as a GCP `startup script`, which gets run on node start. +# That script loops until it can connect to git.arvados.org, and then starts +# sshd so that the Jenkins agent can connect. This avoids the race where Jenkins +# tries to start a job before the GCP outbound routing is working, and fails on +# the first thing it needs internet for, the checkout from git.arvados.org +sudo /bin/systemctl disable sshd diff --git a/jenkins/packer-images/jenkins-image-federation-tests.json b/jenkins/packer-images/jenkins-image-federation-tests.json new file mode 100644 index 0000000..e3d85d2 --- /dev/null +++ b/jenkins/packer-images/jenkins-image-federation-tests.json @@ -0,0 +1,33 @@ +{ + "variables": { + "key_file": "{{env `packerbuildkey`}}" + }, + "builders": [ + { + "type": "googlecompute", + "account_file": "{{user `key_file` }}", + "project_id": "phoenix-project-42-ci", + "source_image": "debian-10-buster-v20200413", + "zone": "us-central1-a", + "disk_size": "10", + "image_name": "federation-jenkins-{{timestamp}}", + "image_family": "jenkins", + "ssh_username": "jenkins" + } + ], + "provisioners": [ + { + "type": "file", + "source": "node-ready.sh", + "destination": "/tmp/node-ready.sh" + }, + { + "type": "shell", + "script": "./jenkins-image-common.sh" + }, + { + "type": "shell", + "script": "./jenkins-image-federation-tests.sh" + } + ] +} diff --git a/jenkins/packer-images/jenkins-image-federation-tests.sh b/jenkins/packer-images/jenkins-image-federation-tests.sh new file mode 100755 index 0000000..ed34703 --- /dev/null +++ b/jenkins/packer-images/jenkins-image-federation-tests.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +sudo su -c "DEBIAN_FRONTEND=noninteractive apt-get install -y docker.io virtualenv curl libcurl4-gnutls-dev build-essential libgnutls28-dev python2.7-dev python3-dev" +sudo usermod -a -G docker jenkins diff --git a/jenkins/packer-images/jenkins-image-k8s-tests.json b/jenkins/packer-images/jenkins-image-k8s-tests.json new file mode 100644 index 0000000..fa76eef --- /dev/null +++ b/jenkins/packer-images/jenkins-image-k8s-tests.json @@ -0,0 +1,39 @@ +{ + "variables": { + "key_file": "{{env `packerbuildkey`}}" + }, + "builders": [ + { + "type": "googlecompute", + "account_file": "{{user `key_file` }}", + "project_id": "phoenix-project-42-ci", + "source_image": "debian-10-buster-v20200413", + "zone": "us-central1-b", + "disk_size": "10", + "image_name": "run-k8s-tests-jenkins-image-{{isotime \"20060102150405\"}}", + "image_family": "jenkins", + "image_licenses": ["projects/vm-options/global/licenses/enable-vmx"], + "ssh_username": "jenkins" + } + ], + "provisioners": [ + { + "type": "file", + "source": "node-ready.sh", + "destination": "/tmp/node-ready.sh" + }, + { + "type": "file", + "source": "1078ECD7.asc", + "destination": "/tmp/1078ECD7.asc" + }, + { + "type": "shell", + "script": "./jenkins-image-common.sh" + }, + { + "type": "shell", + "script": "./jenkins-image-k8s-tests.sh" + } + ] +} diff --git a/jenkins/packer-images/jenkins-image-k8s-tests.sh b/jenkins/packer-images/jenkins-image-k8s-tests.sh new file mode 100755 index 0000000..af409e7 --- /dev/null +++ b/jenkins/packer-images/jenkins-image-k8s-tests.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +set -eo pipefail + +echo "deb http://apt.arvados.org/ buster main" | sudo tee /etc/apt/sources.list.d/arvados.list + +# Install a few dependencies +sudo DEBIAN_FRONTEND=noninteractive apt-get -y --no-install-recommends install gnupg2 wget git default-jdk docker.io netcat + +sudo usermod -a -G docker jenkins + +cat /tmp/1078ECD7.asc | sudo apt-key add - +sudo DEBIAN_FRONTEND=noninteractive apt-get update +# Install Arvados Packages +# the python3 version is currently broken, see #16434, update to python3 when 2.0.3 is out +# python3-arvados-cwl-runner \ +sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \ + python-arvados-cwl-runner \ + python3-arvados-python-client \ + +# Install kubectl + helm +# GCE provides the latest kubectl via apt, automatically +sudo DEBIAN_FRONTEND=noninteractive apt-get install -y kubectl +cd /usr/src +sudo wget https://get.helm.sh/helm-v3.2.1-linux-amd64.tar.gz +sudo tar xzf helm-v3.2.1-linux-amd64.tar.gz +sudo mv linux-amd64/helm /usr/bin/ + +# The rest of this script is what's needed for testing with minikube minikube +sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends dnsmasq + +# Install KVM +sudo DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends qemu-kvm libvirt-clients libvirt-daemon-system + +# Add the jenkins user to the libvirt group +sudo usermod -a -G libvirt jenkins + +# Install minikube +sudo wget -O /usr/local/bin/minikube https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 +sudo chmod +x /usr/local/bin/minikube + +# default to the kvm2 driver *for the jenkins user* (hence, no sudo) +minikube config set driver kvm2 + +sudo DEBIAN_FRONTEND=noninteractive apt-get clean diff --git a/jenkins/packer-images/jenkins-image-tests.json b/jenkins/packer-images/jenkins-image-tests.json new file mode 100644 index 0000000..2356726 --- /dev/null +++ b/jenkins/packer-images/jenkins-image-tests.json @@ -0,0 +1,33 @@ +{ + "variables": { + "key_file": "{{env `packerbuildkey`}}" + }, + "builders": [ + { + "type": "googlecompute", + "account_file": "{{user `key_file` }}", + "project_id": "phoenix-project-42-ci", + "source_image": "debian-10-buster-v20200413", + "zone": "us-central1-b", + "disk_size": "10", + "image_name": "run-tests-jenkins-image-{{isotime \"20060102150405\"}}", + "image_family": "jenkins", + "ssh_username": "jenkins" + } + ], + "provisioners": [ + { + "type": "file", + "source": "node-ready.sh", + "destination": "/tmp/node-ready.sh" + }, + { + "type": "shell", + "script": "./jenkins-image-common.sh" + }, + { + "type": "shell", + "script": "./jenkins-image-tests.sh" + } + ] +} diff --git a/jenkins/packer-images/jenkins-image-tests.sh b/jenkins/packer-images/jenkins-image-tests.sh new file mode 100755 index 0000000..50c873a --- /dev/null +++ b/jenkins/packer-images/jenkins-image-tests.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +set -eo pipefail + +# Install the dependencies for arvados-server +sudo su -c "DEBIAN_FRONTEND=noninteractive apt-get install -y libpam0g-dev golang-1.14" + +# Check out a local copy of the arvados repo so we can use it to install the dependencies +cd /usr/src +sudo git clone arvados.git +cd arvados +/usr/lib/go-1.14/bin/go mod download +sudo /usr/lib/go-1.14/bin/go run ./cmd/arvados-server install -type test + +# Our Jenkins jobs use this directory to store the temporary files for the tests +mkdir /home/jenkins/tmp diff --git a/jenkins/packer-images/node-ready.sh b/jenkins/packer-images/node-ready.sh new file mode 100755 index 0000000..6371496 --- /dev/null +++ b/jenkins/packer-images/node-ready.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +# GCP nodes sometimes have no outbound working network for the first few +# seconds/minutes. +# +# This script will wait until git.arvados.org is reachable. +set -eo pipefail + +# Send all stdout/stderr to the log and to the terminal +exec > >(tee -a /tmp/boot-wait.log) 2>&1 + +# Log a timestamp +date +echo "Starting node-ready.sh" + +while ! /bin/nc -w1 -z git.arvados.org 22; do + echo "Connect failed, waiting 1 second..." + sleep 1 +done +echo "Connected!" + +# All set! Enable and start sshd so jenkins can start the agent... +echo "Re-enabling sshd..." +/bin/systemctl enable ssh || true +echo "Starting sshd..." +/bin/systemctl start ssh || /bin/systemctl status ssh + +echo "Completed node-ready.sh" +# Log a timestamp +date