Simplify write flusher using a time.Ticker.
[arvados.git] / docker / mkimage-debootstrap.sh
1 #!/bin/bash
2 set -e
3
4 variant='minbase'
5 include='iproute,iputils-ping'
6 arch='amd64' # intentionally undocumented for now
7 skipDetection=
8 strictDebootstrap=
9 justTar=
10
11 usage() {
12         echo >&2
13         
14         echo >&2 "usage: $0 [options] repo suite [mirror]"
15         
16         echo >&2
17         echo >&2 'options: (not recommended)'
18         echo >&2 "  -p set an http_proxy for debootstrap"
19         echo >&2 "  -v $variant # change default debootstrap variant"
20         echo >&2 "  -i $include # change default package includes"
21         echo >&2 "  -d # strict debootstrap (do not apply any docker-specific tweaks)"
22         echo >&2 "  -s # skip version detection and tagging (ie, precise also tagged as 12.04)"
23         echo >&2 "     # note that this will also skip adding universe and/or security/updates to sources.list"
24         echo >&2 "  -t # just create a tarball, especially for dockerbrew (uses repo as tarball name)"
25         
26         echo >&2
27         echo >&2 "   ie: $0 username/debian squeeze"
28         echo >&2 "       $0 username/debian squeeze http://ftp.uk.debian.org/debian/"
29         
30         echo >&2
31         echo >&2 "   ie: $0 username/ubuntu precise"
32         echo >&2 "       $0 username/ubuntu precise http://mirrors.melbourne.co.uk/ubuntu/"
33         
34         echo >&2
35         echo >&2 "   ie: $0 -t precise.tar.bz2 precise"
36         echo >&2 "       $0 -t wheezy.tgz wheezy"
37         echo >&2 "       $0 -t wheezy-uk.tar.xz wheezy http://ftp.uk.debian.org/debian/"
38         
39         echo >&2
40 }
41
42 # these should match the names found at http://www.debian.org/releases/
43 debianStable=wheezy
44 debianUnstable=sid
45 # this should match the name found at http://releases.ubuntu.com/
46 ubuntuLatestLTS=precise
47
48 while getopts v:i:a:p:dst name; do
49         case "$name" in
50                 p)
51                         http_proxy="$OPTARG"
52                         ;;
53                 v)
54                         variant="$OPTARG"
55                         ;;
56                 i)
57                         include="$OPTARG"
58                         ;;
59                 a)
60                         arch="$OPTARG"
61                         ;;
62                 d)
63                         strictDebootstrap=1
64                         ;;
65                 s)
66                         skipDetection=1
67                         ;;
68                 t)
69                         justTar=1
70                         ;;
71                 ?)
72                         usage
73                         exit 0
74                         ;;
75         esac
76 done
77 shift $(($OPTIND - 1))
78
79 repo="$1"
80 suite="$2"
81 mirror="${3:-}" # stick to the default debootstrap mirror if one is not provided
82
83 if [ ! "$repo" ] || [ ! "$suite" ]; then
84         usage
85         exit 1
86 fi
87
88 # some rudimentary detection for whether we need to "sudo" our docker calls
89 set +e
90 docker=`which docker.io`
91 if [[ "$docker" == "" ]]; then
92         docker=`which docker`
93 fi
94 set -e
95
96 if $docker version > /dev/null 2>&1; then
97         docker="$docker"
98 elif sudo $docker version > /dev/null 2>&1; then
99         docker="sudo $docker"
100 elif command -v $docker > /dev/null 2>&1; then
101         docker="$docker"
102 else
103         echo >&2 "warning: either docker isn't installed, or your current user cannot run it;"
104         echo >&2 "         this script is not likely to work as expected"
105         sleep 3
106         docker='docker' # give us a command-not-found later
107 fi
108
109 # make sure we have an absolute path to our final tarball so we can still reference it properly after we change directory
110 if [ "$justTar" ]; then
111         if [ ! -d "$(dirname "$repo")" ]; then
112                 echo >&2 "error: $(dirname "$repo") does not exist"
113                 exit 1
114         fi
115         repo="$(cd "$(dirname "$repo")" && pwd -P)/$(basename "$repo")"
116 fi
117
118 # will be filled in later, if [ -z "$skipDetection" ]
119 lsbDist=''
120
121 target="${TMPDIR:-/tmp}/docker-rootfs-debootstrap-$suite-$$-$RANDOM"
122
123 cd "$(dirname "$(readlink -f "$BASH_SOURCE")")"
124 returnTo="$(pwd -P)"
125
126 set -x
127
128 # bootstrap
129 mkdir -p "$target"
130 sudo http_proxy=$http_proxy debootstrap --verbose --variant="$variant" --include="$include" --arch="$arch" "$suite" "$target" "$mirror"
131
132 cd "$target"
133
134 if [ -z "$strictDebootstrap" ]; then
135         # prevent init scripts from running during install/update
136         #  policy-rc.d (for most scripts)
137         echo $'#!/bin/sh\nexit 101' | sudo tee usr/sbin/policy-rc.d > /dev/null
138         sudo chmod +x usr/sbin/policy-rc.d
139         #  initctl (for some pesky upstart scripts)
140         sudo chroot . dpkg-divert --local --rename --add /sbin/initctl
141         sudo ln -sf /bin/true sbin/initctl
142         # see https://github.com/dotcloud/docker/issues/446#issuecomment-16953173
143         
144         # shrink the image, since apt makes us fat (wheezy: ~157.5MB vs ~120MB)
145         sudo chroot . apt-get clean
146         
147         # while we're at it, apt is unnecessarily slow inside containers
148         #  this forces dpkg not to call sync() after package extraction and speeds up install
149         #    the benefit is huge on spinning disks, and the penalty is nonexistent on SSD or decent server virtualization
150         echo 'force-unsafe-io' | sudo tee etc/dpkg/dpkg.cfg.d/02apt-speedup > /dev/null
151         #  we want to effectively run "apt-get clean" after every install to keep images small
152         echo 'DPkg::Post-Invoke {"/bin/rm -f /var/cache/apt/archives/*.deb || true";};' | sudo tee etc/apt/apt.conf.d/no-cache > /dev/null
153         
154         # helpful undo lines for each the above tweaks (for lack of a better home to keep track of them):
155         #  rm /usr/sbin/policy-rc.d
156         #  rm /sbin/initctl; dpkg-divert --rename --remove /sbin/initctl
157         #  rm /etc/dpkg/dpkg.cfg.d/02apt-speedup
158         #  rm /etc/apt/apt.conf.d/no-cache
159         
160         if [ -z "$skipDetection" ]; then
161                 # see also rudimentary platform detection in hack/install.sh
162                 lsbDist=''
163                 if [ -r etc/lsb-release ]; then
164                         lsbDist="$(. etc/lsb-release && echo "$DISTRIB_ID")"
165                 fi
166                 if [ -z "$lsbDist" ] && [ -r etc/debian_version ]; then
167                         lsbDist='Debian'
168                 fi
169                 
170                 case "$lsbDist" in
171                         Debian)
172                                 # add the updates and security repositories
173                                 if [ "$suite" != "$debianUnstable" -a "$suite" != 'unstable' ]; then
174                                         # ${suite}-updates only applies to non-unstable
175                                         sudo sed -i "p; s/ $suite main$/ ${suite}-updates main/" etc/apt/sources.list
176                                         
177                                         # same for security updates
178                                         echo "deb http://security.debian.org/ $suite/updates main" | sudo tee -a etc/apt/sources.list > /dev/null
179                                 fi
180                                 ;;
181                         Ubuntu)
182                                 # add the universe, updates, and security repositories
183                                 sudo sed -i "
184                                         s/ $suite main$/ $suite main universe/; p;
185                                         s/ $suite main/ ${suite}-updates main/; p;
186                                         s/ $suite-updates main/ ${suite}-security main/
187                                 " etc/apt/sources.list
188                                 ;;
189                 esac
190         fi
191 fi
192
193 if [ "$justTar" ]; then
194         # create the tarball file so it has the right permissions (ie, not root)
195         touch "$repo"
196         
197         # fill the tarball
198         sudo tar --numeric-owner -caf "$repo" .
199 else
200         # create the image (and tag $repo:$suite)
201         sudo tar --numeric-owner -c . | $docker import - $repo:$suite
202         
203         # test the image
204         [[ "$(/usr/bin/tty || true)" != "not a tty" ]] && RUN_OPTS="-i -t"
205         $docker run $RUN_OPS $repo:$suite echo success
206         
207         if [ -z "$skipDetection" ]; then
208                 case "$lsbDist" in
209                         Debian)
210                                 if [ "$suite" = "$debianStable" -o "$suite" = 'stable' ] && [ -r etc/debian_version ]; then
211                                         # tag latest
212                                         $docker tag -f $repo:$suite $repo:latest
213                                         
214                                         if [ -r etc/debian_version ]; then
215                                                 # tag the specific debian release version (which is only reasonable to tag on debian stable)
216                                                 ver=$(cat etc/debian_version)
217                                                 $docker tag -f $repo:$suite $repo:$ver
218                                         fi
219                                 fi
220                                 ;;
221                         Ubuntu)
222                                 if [ "$suite" = "$ubuntuLatestLTS" ]; then
223                                         # tag latest
224                                         $docker tag -f $repo:$suite $repo:latest
225                                 fi
226                                 if [ -r etc/lsb-release ]; then
227                                         lsbRelease="$(. etc/lsb-release && echo "$DISTRIB_RELEASE")"
228                                         if [ "$lsbRelease" ]; then
229                                                 # tag specific Ubuntu version number, if available (12.04, etc.)
230                                                 $docker tag -f $repo:$suite $repo:$lsbRelease
231                                         fi
232                                 fi
233                                 ;;
234                 esac
235         fi
236 fi
237
238 # cleanup
239 cd "$returnTo"
240 sudo rm -rf "$target"