package main
import (
+ "bytes"
"context"
- "flag"
"fmt"
"io"
+ "io/ioutil"
"os"
"os/exec"
+ "os/user"
+ "path/filepath"
+ "strings"
- "git.arvados.org/arvados.git/lib/install"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
- "github.com/sirupsen/logrus"
+ "github.com/docker/docker/api/types"
+ "github.com/docker/docker/client"
)
-type build struct{}
-
-func (bld build) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
- logger := ctxlog.New(stderr, "text", "info")
- err := (&builder{
- PackageVersion: "0.0.0",
- logger: logger,
- }).run(context.Background(), prog, args, stdin, stdout, stderr)
- if err != nil {
- logger.WithError(err).Error("failed")
- return 1
+func build(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writer) error {
+ if opts.PackageVersion == "" {
+ var buf bytes.Buffer
+ cmd := exec.CommandContext(ctx, "git", "describe", "--tag", "--dirty")
+ cmd.Stdout = &buf
+ cmd.Stderr = stderr
+ cmd.Dir = opts.SourceDir
+ err := cmd.Run()
+ if err != nil {
+ return fmt.Errorf("git describe: %w", err)
+ }
+ opts.PackageVersion = strings.TrimSpace(buf.String())
+ ctxlog.FromContext(ctx).Infof("version not specified; using %s", opts.PackageVersion)
}
- return 0
-}
-type builder struct {
- PackageVersion string
- SourcePath string
- OutputDir string
- SkipInstall bool
- logger logrus.FieldLogger
-}
+ if opts.PackageChown == "" {
+ whoami, err := user.Current()
+ if err != nil {
+ return fmt.Errorf("user.Current: %w", err)
+ }
+ opts.PackageChown = whoami.Uid + ":" + whoami.Gid
+ }
-func (bldr *builder) run(ctx context.Context, prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) error {
- flags := flag.NewFlagSet("", flag.ContinueOnError)
- flags.StringVar(&bldr.PackageVersion, "package-version", bldr.PackageVersion, "package version")
- flags.StringVar(&bldr.SourcePath, "source", bldr.SourcePath, "source tree location")
- flags.StringVar(&bldr.OutputDir, "output-directory", bldr.OutputDir, "destination directory for new package (default is cwd)")
- flags.BoolVar(&bldr.SkipInstall, "skip-install", bldr.SkipInstall, "skip install step, assume you have already run 'arvados-server install -type package'")
- err := flags.Parse(args)
+ // Build in a tempdir, then move to the desired destination
+ // dir. Otherwise, errors might cause us to leave a mess:
+ // truncated files, files owned by root, etc.
+ _, prog := filepath.Split(os.Args[0])
+ tmpdir, err := ioutil.TempDir(opts.PackageDir, prog+".")
if err != nil {
return err
}
- if len(flags.Args()) > 0 {
- return fmt.Errorf("unrecognized command line arguments: %v", flags.Args())
+ defer os.RemoveAll(tmpdir)
+
+ selfbin, err := os.Readlink("/proc/self/exe")
+ if err != nil {
+ return fmt.Errorf("readlink /proc/self/exe: %w", err)
}
- if !bldr.SkipInstall {
- exitcode := install.Command.RunCommand("arvados-server install", []string{
+ buildImageName := "arvados-package-build-" + opts.TargetOS
+ packageFilename := "arvados-server-easy_" + opts.PackageVersion + "_amd64.deb"
+
+ if ok, err := dockerImageExists(ctx, buildImageName); err != nil {
+ return err
+ } else if !ok || opts.RebuildImage {
+ buildCtrName := strings.Replace(buildImageName, ":", "-", -1)
+ err = dockerRm(ctx, buildCtrName)
+ if err != nil {
+ return err
+ }
+
+ defer dockerRm(ctx, buildCtrName)
+ cmd := exec.CommandContext(ctx, "docker", "run",
+ "--name", buildCtrName,
+ "--tmpfs", "/tmp:exec,mode=01777",
+ "-v", selfbin+":/arvados-package:ro",
+ "-v", opts.SourceDir+":/arvados:ro",
+ opts.TargetOS,
+ "/arvados-package", "_install",
+ "-eatmydata",
"-type", "package",
- "-package-version", bldr.PackageVersion,
- "-source", bldr.SourcePath,
- }, stdin, stdout, stderr)
- if exitcode != 0 {
- return fmt.Errorf("arvados-server install failed: exit code %d", exitcode)
+ "-source", "/arvados",
+ "-package-version", opts.PackageVersion,
+ )
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("docker run: %w", err)
}
+
+ cmd = exec.CommandContext(ctx, "docker", "commit", buildCtrName, buildImageName)
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("docker commit: %w", err)
+ }
+
+ ctxlog.FromContext(ctx).Infof("created docker image %s", buildImageName)
}
- cmd := exec.Command("/var/lib/arvados/bin/gem", "install", "--user", "--no-document", "fpm")
+
+ cmd := exec.CommandContext(ctx, "docker", "run",
+ "--rm",
+ "--tmpfs", "/tmp:exec,mode=01777",
+ "-v", tmpdir+":/pkg",
+ "-v", selfbin+":/arvados-package:ro",
+ "-v", opts.SourceDir+":/arvados:ro",
+ buildImageName,
+ "eatmydata", "/arvados-package", "fpm",
+ "-source", "/arvados",
+ "-package-version", opts.PackageVersion,
+ "-package-dir", "/pkg",
+ "-package-chown", opts.PackageChown,
+ )
cmd.Stdout = stdout
cmd.Stderr = stderr
err = cmd.Run()
if err != nil {
- return fmt.Errorf("gem install fpm: %w", err)
+ return fmt.Errorf("docker run: %w", err)
}
- if _, err := os.Stat("/root/.gem/ruby/2.5.0/gems/fpm-1.11.0/lib/fpm/package/deb.rb"); err == nil {
- // Workaround for fpm bug https://github.com/jordansissel/fpm/issues/1739
- cmd = exec.Command("sed", "-i", `/require "digest"/a require "zlib"`, "/root/.gem/ruby/2.5.0/gems/fpm-1.11.0/lib/fpm/package/deb.rb")
- cmd.Stdout = stdout
- cmd.Stderr = stderr
- err = cmd.Run()
- if err != nil {
- return fmt.Errorf("monkeypatch fpm: %w", err)
- }
+ err = os.Rename(tmpdir+"/"+packageFilename, opts.PackageDir+"/"+packageFilename)
+ if err != nil {
+ return err
}
- // Remove unneeded files. This is much faster than "fpm
- // --exclude X" because fpm copies everything into a staging
- // area before looking at the --exclude args.
- cmd = exec.Command("bash", "-c", "cd /var/www/.gem/ruby && rm -rf */cache */bundler/gems/*/.git */bundler/gems/arvados-*/[^s]* */bundler/gems/arvados-*/s[^d]* */bundler/gems/arvados-*/sdk/[^cr]* */gems/passenger-*/src/cxx* ruby/*/gems/*/ext /var/lib/arvados/go")
+ cmd = exec.CommandContext(ctx, "bash", "-c", "dpkg-scanpackages . | gzip > Packages.gz.tmp && mv Packages.gz.tmp Packages.gz")
cmd.Stdout = stdout
cmd.Stderr = stderr
+ cmd.Dir = opts.PackageDir
err = cmd.Run()
if err != nil {
- return fmt.Errorf("rm -rf [...]: %w", err)
+ return fmt.Errorf("dpkg-scanpackages: %w", err)
}
- format := "deb" // TODO: rpm
+ return nil
+}
- cmd = exec.Command("/root/.gem/ruby/2.5.0/bin/fpm",
- "--name", "arvados-server-easy",
- "--version", bldr.PackageVersion,
- "--input-type", "dir",
- "--output-type", format)
- deps, err := install.ProductionDependencies()
+func dockerRm(ctx context.Context, name string) error {
+ cli, err := client.NewEnvClient()
if err != nil {
return err
}
- for _, pkg := range deps {
- cmd.Args = append(cmd.Args, "--depends", pkg)
+ ctrs, err := cli.ContainerList(ctx, types.ContainerListOptions{All: true, Limit: -1})
+ if err != nil {
+ return err
}
- cmd.Args = append(cmd.Args,
- "--verbose",
- "--deb-use-file-permissions",
- "--rpm-use-file-permissions",
- "/var/lib/arvados",
- "/var/www/.gem",
- "/var/www/.passenger",
- "/var/www/.bundle",
- )
- fmt.Fprintf(stderr, "... %s\n", cmd.Args)
- cmd.Dir = bldr.OutputDir
- cmd.Stdout = stdout
- cmd.Stderr = stderr
- return cmd.Run()
+ for _, ctr := range ctrs {
+ for _, ctrname := range ctr.Names {
+ if ctrname == "/"+name {
+ err = cli.ContainerRemove(ctx, ctr.ID, types.ContainerRemoveOptions{})
+ if err != nil {
+ return fmt.Errorf("error removing container %s: %w", ctr.ID, err)
+ }
+ break
+ }
+ }
+ }
+ return nil
}
package main
import (
+ "context"
+ "flag"
+ "fmt"
+ "io"
"os"
+ "path/filepath"
"git.arvados.org/arvados.git/lib/cmd"
+ "git.arvados.org/arvados.git/lib/install"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
)
var (
"-version": cmd.Version,
"--version": cmd.Version,
- "build": build{},
+ "build": cmdFunc(build),
+ "fpm": cmdFunc(fpm),
+ "testinstall": cmdFunc(testinstall),
+ "_install": install.Command, // internal use
})
)
func main() {
os.Exit(handler.RunCommand(os.Args[0], os.Args[1:], os.Stdin, os.Stdout, os.Stderr))
}
+
+type cmdFunc func(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writer) error
+
+func (cf cmdFunc) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ logger := ctxlog.New(stderr, "text", "info")
+ ctx := ctxlog.Context(context.Background(), logger)
+ opts, err := parseFlags(args)
+ if err != nil {
+ logger.WithError(err).Error("error parsing command line flags")
+ return 1
+ }
+ err = cf(ctx, opts, stdin, stdout, stderr)
+ if err != nil {
+ logger.WithError(err).Error("failed")
+ return 1
+ }
+ return 0
+}
+
+type opts struct {
+ PackageVersion string
+ PackageDir string
+ PackageChown string
+ RebuildImage bool
+ SourceDir string
+ TargetOS string
+}
+
+func parseFlags(args []string) (opts, error) {
+ opts := opts{
+ TargetOS: "debian:10",
+ }
+ flags := flag.NewFlagSet("", flag.ContinueOnError)
+ flags.StringVar(&opts.PackageVersion, "package-version", opts.PackageVersion, "package version to build/test, like \"1.2.3\"")
+ flags.StringVar(&opts.SourceDir, "source", opts.SourceDir, "arvados source tree location")
+ flags.StringVar(&opts.PackageDir, "package-dir", opts.PackageDir, "destination directory for new package (default is cwd)")
+ flags.StringVar(&opts.PackageChown, "package-chown", opts.PackageChown, "desired uid:gid for new package (default is current user:group)")
+ flags.StringVar(&opts.TargetOS, "target-os", opts.TargetOS, "target operating system vendor:version")
+ flags.BoolVar(&opts.RebuildImage, "rebuild-image", opts.RebuildImage, "rebuild docker image(s) instead of using existing")
+ err := flags.Parse(args)
+ if err != nil {
+ return opts, err
+ }
+ if len(flags.Args()) > 0 {
+ return opts, fmt.Errorf("unrecognized command line arguments: %v", flags.Args())
+ }
+ if opts.SourceDir == "" {
+ d, err := os.Getwd()
+ if err != nil {
+ return opts, fmt.Errorf("Getwd: %w", err)
+ }
+ opts.SourceDir = d
+ }
+ opts.PackageDir = filepath.Clean(opts.PackageDir)
+ return opts, nil
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "os"
+ "os/exec"
+ "path/filepath"
+
+ "git.arvados.org/arvados.git/lib/install"
+)
+
+func fpm(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writer) error {
+ var chownUid, chownGid int
+ if opts.PackageChown != "" {
+ _, err := fmt.Sscanf(opts.PackageChown, "%d:%d", &chownUid, &chownGid)
+ if err != nil {
+ return fmt.Errorf("invalid value %q for PackageChown: %w", opts.PackageChown, err)
+ }
+ }
+
+ exitcode := install.Command.RunCommand("arvados-server install", []string{
+ "-type", "package",
+ "-package-version", opts.PackageVersion,
+ "-source", opts.SourceDir,
+ }, stdin, stdout, stderr)
+ if exitcode != 0 {
+ return fmt.Errorf("arvados-server install failed: exit code %d", exitcode)
+ }
+
+ cmd := exec.Command("/var/lib/arvados/bin/gem", "install", "--user", "--no-document", "fpm")
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err := cmd.Run()
+ if err != nil {
+ return fmt.Errorf("gem install fpm: %w", err)
+ }
+
+ if _, err := os.Stat("/root/.gem/ruby/2.5.0/gems/fpm-1.11.0/lib/fpm/package/deb.rb"); err == nil {
+ // Workaround for fpm bug https://github.com/jordansissel/fpm/issues/1739
+ cmd = exec.Command("sed", "-i", `/require "digest"/a require "zlib"`, "/root/.gem/ruby/2.5.0/gems/fpm-1.11.0/lib/fpm/package/deb.rb")
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("monkeypatch fpm: %w", err)
+ }
+ }
+
+ // Remove unneeded files. This is much faster than "fpm
+ // --exclude X" because fpm copies everything into a staging
+ // area before looking at the --exclude args.
+ cmd = exec.Command("bash", "-c", "cd /var/www/.gem/ruby && rm -rf */cache */bundler/gems/*/.git */bundler/gems/arvados-*/[^s]* */bundler/gems/arvados-*/s[^d]* */bundler/gems/arvados-*/sdk/[^cr]* */gems/passenger-*/src/cxx* ruby/*/gems/*/ext /var/lib/arvados/go")
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("rm -rf [...]: %w", err)
+ }
+
+ format := "deb" // TODO: rpm
+ pkgfile := filepath.Join(opts.PackageDir, "arvados-server-easy_"+opts.PackageVersion+"_amd64."+format)
+
+ cmd = exec.Command("/root/.gem/ruby/2.5.0/bin/fpm",
+ "--package", pkgfile,
+ "--name", "arvados-server-easy",
+ "--version", opts.PackageVersion,
+ "--input-type", "dir",
+ "--output-type", format)
+ deps, err := install.ProductionDependencies()
+ if err != nil {
+ return err
+ }
+ for _, pkg := range deps {
+ cmd.Args = append(cmd.Args, "--depends", pkg)
+ }
+ cmd.Args = append(cmd.Args,
+ "--verbose",
+ "--deb-use-file-permissions",
+ "--rpm-use-file-permissions",
+ "/var/lib/arvados",
+ "/var/www/.gem",
+ "/var/www/.passenger",
+ "/var/www/.bundle",
+ )
+ fmt.Fprintf(stderr, "... %s\n", cmd.Args)
+ cmd.Dir = opts.PackageDir
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("fpm: %w", err)
+ }
+
+ if opts.PackageChown != "" {
+ err = os.Chown(pkgfile, chownUid, chownGid)
+ if err != nil {
+ return fmt.Errorf("chown %s: %w", pkgfile, err)
+ }
+ }
+
+ cmd = exec.Command("ls", "-l", pkgfile)
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ _ = cmd.Run()
+
+ return nil
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+
+ "github.com/docker/docker/api/types"
+ "github.com/docker/docker/client"
+)
+
+// sourcesfile=/tmp/sources.conf.d-arvados
+// echo >$sourcesfile "deb [trusted=yes] file:///pkg ./"
+
+// installimage="arvados-installpackage-${osbase}"
+// if [[ "${opts[force-installimage]}" || -z "$(docker images --format {{.Repository}} "${installimage}")" ]]; then
+// echo >&2 building ${installimage}...
+// installctr=${installimage/:/-}
+// docker rm "${installctr}" || true
+// docker run -it \
+// --name "${installctr}" \
+// --tmpfs /tmp \
+// -v /tmp/pkg:/pkg:ro \
+// -v ${sourcesfile}:/etc/apt/sources.list.d/arvados-local.list:ro \
+// --env DEBIAN_FRONTEND=noninteractive \
+// "${osbase}" \
+// bash -c 'apt update && apt install -y eatmydata && eatmydata apt install -y arvados-server-easy postgresql && eatmydata apt remove -y arvados-server-easy'
+// docker commit "${installctr}" "${installimage}"
+// docker rm "${installctr}"
+// installctr=
+// fi
+
+func testinstall(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writer) error {
+ if opts.PackageVersion != "" {
+ return errors.New("not implemented: package version was specified, but I only know how to test the latest version in pkgdir")
+ }
+ depsImageName := "arvados-package-deps-" + opts.TargetOS
+ depsCtrName := strings.Replace(depsImageName, ":", "-", -1)
+
+ _, prog := filepath.Split(os.Args[0])
+ tmpdir, err := ioutil.TempDir("", prog+".")
+ if err != nil {
+ return fmt.Errorf("TempDir: %w", err)
+ }
+ defer os.RemoveAll(tmpdir)
+
+ sourcesFile := tmpdir + "/arvados-local.list"
+ err = ioutil.WriteFile(sourcesFile, []byte("deb [trusted=yes] file:///pkg ./\n"), 0644)
+ if err != nil {
+ return fmt.Errorf("Write %s: %w", sourcesFile, err)
+ }
+
+ if exists, err := dockerImageExists(ctx, depsImageName); err != nil {
+ return err
+ } else if !exists || opts.RebuildImage {
+ err = dockerRm(ctx, depsCtrName)
+ if err != nil {
+ return err
+ }
+ defer dockerRm(ctx, depsCtrName)
+ cmd := exec.CommandContext(ctx, "docker", "run",
+ "--name", depsCtrName,
+ "--tmpfs", "/tmp:exec,mode=01777",
+ "-v", opts.PackageDir+":/pkg:ro",
+ "-v", sourcesFile+":/etc/apt/sources.list.d/arvados-local.list:ro",
+ "--env", "DEBIAN_FRONTEND=noninteractive",
+ opts.TargetOS,
+ "bash", "-c", `
+set -e
+apt-get update
+apt-get install -y eatmydata
+eatmydata apt-get install -y --no-install-recommends arvados-server-easy postgresql
+eatmydata apt-get remove -y arvados-server-easy
+`)
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("docker run: %w", err)
+ }
+
+ cmd = exec.CommandContext(ctx, "docker", "commit", depsCtrName, depsImageName)
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("docker commit: %w", err)
+ }
+ }
+
+ cmd := exec.CommandContext(ctx, "docker", "run", "--rm",
+ "--tmpfs", "/tmp:exec,mode=01777",
+ "-v", opts.PackageDir+":/pkg:ro",
+ "-v", sourcesFile+":/etc/apt/sources.list.d/arvados-local.list:ro",
+ "--env", "DEBIAN_FRONTEND=noninteractive",
+ depsImageName,
+ "bash", "-c", `
+set -e
+PATH="/var/lib/arvados/bin:$PATH"
+apt-get update
+eatmydata apt-get install --reinstall -y --no-install-recommends arvados-server-easy
+apt-get -y autoremove
+/etc/init.d/postgresql start
+arvados-server init -cluster-id x1234
+exec arvados-server boot -listen-host 0.0.0.0 -shutdown
+`)
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("docker run: %w", err)
+ }
+ return nil
+}
+
+func dockerImageExists(ctx context.Context, name string) (bool, error) {
+ cli, err := client.NewEnvClient()
+ if err != nil {
+ return false, err
+ }
+ imgs, err := cli.ImageList(ctx, types.ImageListOptions{All: true})
+ if err != nil {
+ return false, err
+ }
+ for _, img := range imgs {
+ for _, tag := range img.RepoTags {
+ if tag == name {
+ return true, nil
+ }
+ }
+ }
+ return false, nil
+}
projects / arvados.git / commitdiff