Merged branch 1741-internal-git-repo into 2333-crunch-dispatch-token-management

diff --git a/doc/README.textile b/doc/README.textile
index 23a82fe5b115e25468666e23980008d497ef7d50..f88d30435c20ff1af435e8e37f9340f80a03622d 100644 (file)
--- a/doc/README.textile
+++ b/doc/README.textile
@@ -18,6 +18,12 @@ h2. 1. Generate HTML pages
 arvados/doc$ rake
 </pre>
 
+Alternately, to make the documentation browsable on the local filesystem:
+
+<pre>
+arvados/doc$ rake generate baseurl=$PWD/.site
+</pre>
+
 h2. 2. Preview HTML pages
 
 <pre>

diff --git a/sdk/cli/bin/arv-run-pipeline-instance b/sdk/cli/bin/arv-run-pipeline-instance
index 9f1663d1a32656c688bcd057dedb9a074341912e..e0e002fc7f50ad8cafc8ebb447676646ab46926a 100755 (executable)
--- a/sdk/cli/bin/arv-run-pipeline-instance
+++ b/sdk/cli/bin/arv-run-pipeline-instance
@@ -433,6 +433,7 @@ class WhRunPipelineInstance
           job = JobCache.create({:script => c[:script],
                             :script_parameters => c[:script_parameters],
                             :script_version => c[:script_version],
+                            :repository => c[:repository],
                             :minimum_script_version => c[:minimum_script_version],
                             :exclude_script_versions => c[:exclude_minimum_script_versions],
                             :nondeterministic => c[:nondeterministic],

diff --git a/services/api/app/controllers/arvados/v1/jobs_controller.rb b/services/api/app/controllers/arvados/v1/jobs_controller.rb
index 07daf71eb6caa5ae022b0bff9ad37f66e9231cc8..b35bc4e44f8f38edb57c659ba2f21e84c12f05d2 100644 (file)
--- a/services/api/app/controllers/arvados/v1/jobs_controller.rb
+++ b/services/api/app/controllers/arvados/v1/jobs_controller.rb
@@ -49,7 +49,6 @@ class Arvados::V1::JobsController < ApplicationController
     end
 
     # Don't pass these on to activerecord
-    resource_attrs.delete(:repository)
     resource_attrs.delete(:minimum_script_version)
     resource_attrs.delete(:exclude_script_versions)
     resource_attrs.delete(:no_reuse)

diff --git a/services/api/app/models/commit.rb b/services/api/app/models/commit.rb
index 49541248f28cf5b4f677710aa30f03991a2fa6af..6239011a1bda462dca713feec0bc18250025e54b 100644 (file)
--- a/services/api/app/models/commit.rb
+++ b/services/api/app/models/commit.rb
@@ -139,6 +139,7 @@ class Commit < ActiveRecord::Base
       next if repo.match /^\./
       git_dir = File.join(@gitdirbase,
                           repo.match(/\.git$/) ? repo : File.join(repo, '.git'))
+      next if git_dir == Rails.configuration.git_internal_dir
       repo_name = repo.sub(/\.git$/, '')
       @repositories[repo_name] = {git_dir: git_dir}
     end

diff --git a/services/api/app/models/job.rb b/services/api/app/models/job.rb
index 6e3f90c38f7046550fb88d8203670158ee7d167b..4af3fa84e94194b555c86cb56476c069bfbb59f7 100644 (file)
--- a/services/api/app/models/job.rb
+++ b/services/api/app/models/job.rb
@@ -37,6 +37,7 @@ class Job < ArvadosModel
     t.add :log_stream_href
     t.add :log_buffer
     t.add :nondeterministic
+    t.add :repository
   end
 
   def assert_finished

diff --git a/services/api/config/application.default.yml b/services/api/config/application.default.yml
index e46ff5dd187c2777060d1bd9dba16132e875cfa9..298b001c547ae7cba2df994a73982040b022a311 100644 (file)
--- a/services/api/config/application.default.yml
+++ b/services/api/config/application.default.yml
@@ -45,7 +45,17 @@ common:
   secret_token: ~
   uuid_prefix: <%= Digest::MD5.hexdigest(`hostname`).to_i(16).to_s(36)[0..4] %>
 
-  git_repositories_dir: /var/cache/git
+  # Git repositories must be readable by api server, or you won't be
+  # able to submit crunch jobs. To pass the test suites, put a clone
+  # of the arvados tree in {git_repositories_dir}/arvados.git or
+  # {git_repositories_dir}/arvados/.git
+  git_repositories_dir: /var/lib/arvados/git
+
+  # This is a (bare) repository that stores commits used in jobs.  When a job
+  # runs, the source commits are first fetched into this repository, then this
+  # repository is used to deploy to compute nodes.  This should NOT be a
+  # subdirectory of {git_repositiories_dir}.
+  git_internal_dir: /var/lib/arvados/internal.git
 
   # :none or :slurm_immediate
   crunch_job_wrapper: :none

diff --git a/services/api/config/application.yml.example b/services/api/config/application.yml.example
index 488c4bd3e6cff843ecac18f3a4142ca99d2182cf..9162fc444585d9fa1e59ade9f0df7e144f81cbda 100644 (file)
--- a/services/api/config/application.yml.example
+++ b/services/api/config/application.yml.example
@@ -37,10 +37,5 @@ test:
   secret_token: <%= rand(2**512).to_s(36) %>
 
 common:
-
-  # Git repositories must be readable by api server, or you won't be
-  # able to submit crunch jobs. To pass the test suites, put a clone
-  # of the arvados tree in {git_repositories_dir}/arvados.git or
-  # {git_repositories_dir}/arvados/.git
-  #
   #git_repositories_dir: /var/cache/git
+  #git_internal_dir: /var/cache/arvados/internal.git

diff --git a/services/api/db/migrate/20140321191343_add_repository_column_to_job.rb b/services/api/db/migrate/20140321191343_add_repository_column_to_job.rb
new file mode 100644 (file)
index 0000000..e1ebb2e
--- /dev/null
+++ b/services/api/db/migrate/20140321191343_add_repository_column_to_job.rb
@@ -0,0 +1,9 @@
+class AddRepositoryColumnToJob < ActiveRecord::Migration
+  def up
+    add_column :jobs, :repository, :string
+  end
+
+  def down
+    remove_column :jobs, :repository
+  end
+end

diff --git a/services/api/db/schema.rb b/services/api/db/schema.rb
index 4d638b5430b4b60d62709569251f32d9c6bd03b7..97e6e9c0cd3035b66675f9aa8b9352c1a165d6e1 100644 (file)
--- a/services/api/db/schema.rb
+++ b/services/api/db/schema.rb
@@ -11,7 +11,7 @@
 #
 # It's strongly recommended to check this file into your version control system.
 
-ActiveRecord::Schema.define(:version => 20140319160547) do
+ActiveRecord::Schema.define(:version => 20140321191343) do
 
   create_table "api_client_authorizations", :force => true do |t|
     t.string   "api_token",                                           :null => false
@@ -190,6 +190,7 @@ ActiveRecord::Schema.define(:version => 20140319160547) do
     t.text     "tasks_summary"
     t.text     "runtime_constraints"
     t.boolean  "nondeterministic"
+    t.string   "repository"
   end
 
   add_index "jobs", ["created_at"], :name => "index_jobs_on_created_at"

diff --git a/services/api/script/crunch-dispatch.rb b/services/api/script/crunch-dispatch.rb
index c35c5dec54874c97b22e5c1b366050946f6e2f60..d9db69f6502a87ca22c5565c149806b6e5da6198 100755 (executable)
--- a/services/api/script/crunch-dispatch.rb
+++ b/services/api/script/crunch-dispatch.rb
@@ -150,27 +150,32 @@ class Dispatcher
         raise "No CRUNCH_JOB_BIN env var, and crunch-job not in path."
       end
 
+      require 'shellwords'
+
+      arvados_internal = Rails.configuration.git_internal_dir
+      if not File.exists? arvados_internal
+        $stderr.puts `mkdir -p #{arvados_internal.shellescape} && cd #{arvados_internal.shellescape} && git init --bare`
+      end
+
+      src_repo = File.join(Rails.configuration.git_repositories_dir, job.repository + '.git')
+      src_repo = File.join(Rails.configuration.git_repositories_dir, job.repository, '.git') unless File.exists? src_repo
+
+      unless src_repo
+        $stderr.puts "dispatch: #{File.join Rails.configuration.git_repositories_dir, job.repository} doesn't exist"
+        sleep 1
+        untake(job)
+        next
+      end
+
+      $stderr.puts `cd #{arvados_internal.shellescape} && git fetch --no-tags #{src_repo.shellescape} && git tag #{job.uuid.shellescape} #{job.script_version.shellescape}`
+
       cmd_args << crunch_job_bin
       cmd_args << '--job-api-token'
       cmd_args << job_auth.api_token
       cmd_args << '--job'
       cmd_args << job.uuid
-
-      commit = Commit.where(sha1: job.script_version).first
-      if commit
-        cmd_args << '--git-dir'
-        if File.exists?(File.
-                        join(Rails.configuration.git_repositories_dir,
-                             commit.repository_name + '.git'))
-          cmd_args << File.
-            join(Rails.configuration.git_repositories_dir,
-                 commit.repository_name + '.git')
-        else
-          cmd_args << File.
-            join(Rails.configuration.git_repositories_dir,
-                 commit.repository_name, '.git')
-        end
-      end
+      cmd_args << '--git-dir'
+      cmd_args << arvados_internal
 
       $stderr.puts "dispatch: #{cmd_args.join ' '}"