19709: Apply pending rails migrations at service start/restart.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curii.com>

diff --git a/lib/boot/rails_db.go b/lib/boot/rails_db.go
new file mode 100644 (file)
index 0000000..3f81511
--- /dev/null
+++ b/lib/boot/rails_db.go
@@ -0,0 +1,89 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package boot
+
+import (
+       "context"
+       "io/fs"
+       "os"
+       "path/filepath"
+       "strings"
+
+       "git.arvados.org/arvados.git/lib/controller/dblock"
+       "git.arvados.org/arvados.git/lib/ctrlctx"
+)
+
+type railsDatabase struct{}
+
+func (runner railsDatabase) String() string {
+       return "railsDatabase"
+}
+
+func (runner railsDatabase) Run(ctx context.Context, fail func(error), super *Supervisor) error {
+       err := super.wait(ctx, runPostgreSQL{}, installPassenger{src: "services/api"})
+       if err != nil {
+               return err
+       }
+
+       // determine path to installed rails app or source tree
+       var appdir string
+       if super.ClusterType == "production" {
+               appdir = "/var/lib/arvados/railsapi"
+       } else {
+               appdir = filepath.Join(super.SourcePath, "services/api")
+       }
+
+       // list versions in db/migrate/{version}_{name}.rb
+       todo := map[string]bool{}
+       fs.WalkDir(os.DirFS(appdir), "db/migrate", func(path string, d fs.DirEntry, err error) error {
+               if cut := strings.Index(d.Name(), "_"); cut > 0 && strings.HasSuffix(d.Name(), ".rb") {
+                       todo[d.Name()[:cut]] = true
+               }
+               return nil
+       })
+
+       // read schema_migrations table (list of migrations already
+       // applied) and remove those entries from todo
+       dbconnector := ctrlctx.DBConnector{PostgreSQL: super.cluster.PostgreSQL}
+       defer dbconnector.Close()
+       db, err := dbconnector.GetDB(ctx)
+       if err != nil {
+               return err
+       }
+       rows, err := db.QueryContext(ctx, `SELECT version FROM schema_migrations`)
+       if err != nil {
+               if super.ClusterType == "production" {
+                       return err
+               }
+               super.logger.WithError(err).Info("schema_migrations query failed, trying db:setup")
+               return super.RunProgram(ctx, "services/api", runOptions{env: railsEnv}, "bundle", "exec", "rake", "db:setup")
+       }
+       for rows.Next() {
+               var v string
+               err = rows.Scan(&v)
+               if err != nil {
+                       return err
+               }
+               delete(todo, v)
+       }
+       err = rows.Close()
+       if err != nil {
+               return err
+       }
+
+       // if nothing remains in todo, all available migrations are
+       // done, so return without running any [relatively slow]
+       // ruby/rake commands
+       if len(todo) == 0 {
+               return nil
+       }
+
+       super.logger.Infof("%d migrations pending", len(todo))
+       if !dblock.RailsMigrations.Lock(ctx, dbconnector.GetDB) {
+               return context.Canceled
+       }
+       defer dblock.RailsMigrations.Unlock()
+       return super.RunProgram(ctx, appdir, runOptions{env: railsEnv}, "bundle", "exec", "rake", "db:migrate")
+}

diff --git a/lib/boot/seed.go b/lib/boot/seed.go
deleted file mode 100644 (file)
index b43d907..0000000
--- a/lib/boot/seed.go
+++ /dev/null
@@ -1,31 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package boot
-
-import (
-       "context"
-)
-
-// Populate a blank database with arvados tables and seed rows.
-type seedDatabase struct{}
-
-func (seedDatabase) String() string {
-       return "seedDatabase"
-}
-
-func (seedDatabase) Run(ctx context.Context, fail func(error), super *Supervisor) error {
-       err := super.wait(ctx, runPostgreSQL{}, installPassenger{src: "services/api"})
-       if err != nil {
-               return err
-       }
-       if super.ClusterType == "production" {
-               return nil
-       }
-       err = super.RunProgram(ctx, "services/api", runOptions{env: railsEnv}, "bundle", "exec", "rake", "db:setup")
-       if err != nil {
-               return err
-       }
-       return nil
-}

diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index ca88653fa643e56a93cdb8eb2136cacbf403044f..0f0600f181d7e4371687f921f0f2ca318db80546 100644 (file)
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -356,20 +356,24 @@ func (super *Supervisor) runCluster() error {
                createCertificates{},
                runPostgreSQL{},
                runNginx{},
-               runServiceCommand{name: "controller", svc: super.cluster.Services.Controller, depends: []supervisedTask{seedDatabase{}}},
+               railsDatabase{},
+               runServiceCommand{name: "controller", svc: super.cluster.Services.Controller, depends: []supervisedTask{railsDatabase{}}},
                runServiceCommand{name: "git-httpd", svc: super.cluster.Services.GitHTTP},
                runServiceCommand{name: "health", svc: super.cluster.Services.Health},
                runServiceCommand{name: "keepproxy", svc: super.cluster.Services.Keepproxy, depends: []supervisedTask{runPassenger{src: "services/api"}}},
                runServiceCommand{name: "keepstore", svc: super.cluster.Services.Keepstore},
                runServiceCommand{name: "keep-web", svc: super.cluster.Services.WebDAV},
-               runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{seedDatabase{}}},
+               runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{railsDatabase{}}},
                installPassenger{src: "services/api", varlibdir: "railsapi"},
-               runPassenger{src: "services/api", varlibdir: "railsapi", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, seedDatabase{}, installPassenger{src: "services/api", varlibdir: "railsapi"}}},
-               seedDatabase{},
+               runPassenger{src: "services/api", varlibdir: "railsapi", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{
+                       createCertificates{},
+                       installPassenger{src: "services/api", varlibdir: "railsapi"},
+                       railsDatabase{},
+               }},
        }
        if !super.NoWorkbench1 {
                tasks = append(tasks,
-                       installPassenger{src: "apps/workbench", varlibdir: "workbench1", depends: []supervisedTask{seedDatabase{}}}, // dependency ensures workbench doesn't delay api install/startup
+                       installPassenger{src: "apps/workbench", varlibdir: "workbench1", depends: []supervisedTask{railsDatabase{}}}, // dependency ensures workbench doesn't delay api install/startup
                        runPassenger{src: "apps/workbench", varlibdir: "workbench1", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench", varlibdir: "workbench1"}}},
                )
        }

diff --git a/lib/controller/dblock/dblock.go b/lib/controller/dblock/dblock.go
index ad2733abfa36df82c72c4aa3c7a6c090c6496efb..c59bcef0b272e3c5618f6b3e4119c1455c0dbf73 100644 (file)
--- a/lib/controller/dblock/dblock.go
+++ b/lib/controller/dblock/dblock.go
@@ -22,6 +22,7 @@ var (
        KeepBalanceService = &DBLocker{key: 10003} // keep-balance service in periodic-sweep loop
        KeepBalanceActive  = &DBLocker{key: 10004} // keep-balance sweep in progress (either -once=true or service loop)
        Dispatch           = &DBLocker{key: 10005} // any dispatcher running
+       RailsMigrations    = &DBLocker{key: 10006}
        retryDelay         = 5 * time.Second
 )
 

diff --git a/lib/ctrlctx/db.go b/lib/ctrlctx/db.go
index 2a05096ce18b7430e7e1e487dd5d710024ac9193..b711b3e650cb2f1c19825c71683cc3817b9abd59 100644 (file)
--- a/lib/ctrlctx/db.go
+++ b/lib/ctrlctx/db.go
@@ -168,8 +168,20 @@ func (dbc *DBConnector) GetDB(ctx context.Context) (*sqlx.DB, error) {
        }
        if err := db.Ping(); err != nil {
                ctxlog.FromContext(ctx).WithError(err).Error("postgresql connect succeeded but ping failed")
+               db.Close()
                return nil, errDBConnection
        }
        dbc.pgdb = db
        return db, nil
 }
+
+func (dbc *DBConnector) Close() error {
+       dbc.mtx.Lock()
+       defer dbc.mtx.Unlock()
+       var err error
+       if dbc.pgdb != nil {
+               err = dbc.pgdb.Close()
+               dbc.pgdb = nil
+       }
+       return err
+}