11807: Migrate old records in jobs table from YAML to JSON.

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curoverse.com>

diff --git a/services/api/db/migrate/20170628185847_jobs_yaml_to_json.rb b/services/api/db/migrate/20170628185847_jobs_yaml_to_json.rb
new file mode 100644 (file)
index 0000000..705ec55
--- /dev/null
+++ b/services/api/db/migrate/20170628185847_jobs_yaml_to_json.rb
@@ -0,0 +1,17 @@
+require 'migrate_yaml_to_json'
+
+class JobsYamlToJson < ActiveRecord::Migration
+  def up
+    [
+      'components',
+      'script_parameters',
+      'runtime_constraints',
+      'tasks_summary',
+    ].each do |column|
+      MigrateYAMLToJSON.migrate("jobs", column)
+    end
+  end
+
+  def down
+  end
+end

diff --git a/services/api/db/structure.sql b/services/api/db/structure.sql
index 3e1fa3fae4be514e5d47761ca616ef072142fae3..66f6e71f44b2deed00a7ebf5932a1e3b5ef85676 100644 (file)
--- a/services/api/db/structure.sql
+++ b/services/api/db/structure.sql
@@ -2785,3 +2785,5 @@ INSERT INTO schema_migrations (version) VALUES ('20170419173712');
 
 INSERT INTO schema_migrations (version) VALUES ('20170419175801');
 
 
 INSERT INTO schema_migrations (version) VALUES ('20170419175801');
 

+INSERT INTO schema_migrations (version) VALUES ('20170628185847');
+

diff --git a/services/api/lib/migrate_yaml_to_json.rb b/services/api/lib/migrate_yaml_to_json.rb
new file mode 100644 (file)
index 0000000..fbfc603
--- /dev/null
+++ b/services/api/lib/migrate_yaml_to_json.rb
@@ -0,0 +1,28 @@
+module MigrateYAMLToJSON
+  def self.migrate(table, column)
+    conn = ActiveRecord::Base.connection
+    n = conn.update(
+      "UPDATE #{table} SET #{column}=$1 WHERE #{column}=$2",
+      "#{table}.#{column} convert YAML to JSON",
+      [[nil, "{}"], [nil, "--- {}\n"]])
+    Rails.logger.info("#{table}.#{column}: #{n} rows updated using empty hash")
+    finished = false
+    while !finished
+      n = 0
+      conn.exec_query(
+        "SELECT id, #{column} FROM #{table} WHERE #{column} LIKE $1 LIMIT 100",
+        "#{table}.#{column} check for YAML",
+        [[nil, '---%']],
+      ).rows.map do |id, yaml|
+        n += 1
+        json = SafeJSON.dump(YAML.load(yaml))
+        conn.exec_query(
+          "UPDATE #{table} SET #{column}=$1 WHERE id=$2 AND #{column}=$3",
+          "#{table}.#{column} convert YAML to JSON",
+          [[nil, json], [nil, id], [nil, yaml]])
+      end
+      Rails.logger.info("#{table}.#{column}: #{n} rows updated")
+      finished = (n == 0)
+    end
+  end
+end