X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/789e479916a89e57cdf1d65c49923449c4fb1bcf..cac035d7a171b90f0dc714d17b500b4062c0c58b:/services/api/script/crunch-dispatch.rb?ds=inline

diff --git a/services/api/script/crunch-dispatch.rb b/services/api/script/crunch-dispatch.rb
index 00f3fa9fca..ebd5165669 100755
--- a/services/api/script/crunch-dispatch.rb
+++ b/services/api/script/crunch-dispatch.rb
@@ -93,6 +93,9 @@ class Dispatcher
   def slurm_status
     slurm_nodes = {}
     each_slurm_line("sinfo", "%t") do |hostname, state|
+      # Treat nodes in idle* state as down, because the * means that slurm
+      # hasn't been able to communicate with it recently.
+      state.sub!(/^idle\*/, "down")
       state.sub!(/\W+$/, "")
       state = "down" unless %w(idle alloc down).include?(state)
       slurm_nodes[hostname] = {state: state, job: nil}
@@ -198,13 +201,19 @@ class Dispatcher
               event_type: 'dispatch',
               owner_uuid: job.owner_uuid,
               summary: message,
-              properties: {"text" => message}).save
+              properties: {"text" => message}).save!
     rescue
       $stderr.puts "dispatch: log.create failed"
     end
-    job.state = "Failed"
-    if not job.save
-      $stderr.puts "dispatch: job.save failed"
+
+    begin
+      job.lock @authorizations[job.uuid].user.uuid
+      job.state = "Failed"
+      if not job.save
+        $stderr.puts "dispatch: save failed setting job #{job.uuid} to failed"
+      end
+    rescue ArvadosModel::AlreadyLockedError
+      $stderr.puts "dispatch: tried to mark job #{job.uuid} as failed but it was already locked by someone else"
     end
   end
 
@@ -282,8 +291,10 @@ class Dispatcher
 
       git = "git --git-dir=#{arvados_internal.shellescape}"
 
-      @have_commits ||= {}
-      if !@have_commits[job.script_version]
+      # @fetched_commits[V]==true if we know commit V exists in the
+      # arvados_internal git repository.
+      @fetched_commits ||= {}
+      if !@fetched_commits[job.script_version]
 
         repo_root = Rails.configuration.git_repositories_dir
         src_repo = File.join(repo_root, job.repository + '.git')
@@ -300,41 +311,47 @@ class Dispatcher
         unless $? == 0 and commit_rev == job.script_version
           # commit does not exist in internal repository, so import the source repository using git fetch-pack
           cmd = "#{git} fetch-pack --no-progress --all #{src_repo.shellescape}"
-          $stderr.puts cmd
+          $stderr.puts "dispatch: #{cmd}"
           $stderr.puts `#{cmd}`
           unless $? == 0
             fail_job job, "git fetch-pack failed"
             next
           end
         end
-        @have_commits[job.script_version] = true
+        @fetched_commits[job.script_version] = true
       end
 
-      @have_tags ||= {}
-      if not @have_tags[job.uuid]
-        # check if the commit needs to be tagged with this job uuid
-        tag_rev = `#{git} rev-list -n1 #{job.uuid.shellescape} 2>/dev/null`.chomp
-        if $? != 0
-          # no job tag found, so create one
-          cmd = "#{git} tag #{job.uuid.shellescape} #{job.script_version.shellescape}"
-          $stderr.puts cmd
-          $stderr.puts `#{cmd}`
-          unless $? == 0
-            fail_job job, "git tag failed"
-            next
-          end
-        else
-          # job tag found, check that it has the expected revision
-          unless tag_rev == job.script_version
-            # Uh oh, the tag doesn't point to the revision we were expecting.
-            # Someone has been monkeying with the job record and/or git.
-            fail_job job, "Existing tag #{job.uuid} points to commit #{tag_rev} but expected commit #{job.script_version}"
+      # @job_tags[J]==V if we know commit V has been tagged J in the
+      # arvados_internal repository. (J is a job UUID, V is a commit
+      # sha1.)
+      @job_tags ||= {}
+      if not @job_tags[job.uuid]
+        cmd = "#{git} tag #{job.uuid.shellescape} #{job.script_version.shellescape} 2>/dev/null"
+        $stderr.puts "dispatch: #{cmd}"
+        $stderr.puts `#{cmd}`
+        unless $? == 0
+          # git tag failed.  This may be because the tag already exists, so check for that.
+          tag_rev = `#{git} rev-list -n1 #{job.uuid.shellescape}`.chomp
+          if $? == 0
+            # We got a revision back
+            if tag_rev != job.script_version
+              # Uh oh, the tag doesn't point to the revision we were expecting.
+              # Someone has been monkeying with the job record and/or git.
+              fail_job job, "Existing tag #{job.uuid} points to commit #{tag_rev} but expected commit #{job.script_version}"
+              next
+            end
+            # we're okay (fall through to setting @job_tags below)
+          else
+            # git rev-list failed for some reason.
+            fail_job job, "'git tag' for #{job.uuid} failed but did not find any existing tag using 'git rev-list'"
             next
           end
         end
-        @have_tags[job.uuid] = job.script_version
-      elsif @have_tags[job.uuid] != job.script_version
-        fail_job job, "Existing tag #{job.uuid} points to commit #{@have_tags[job.uuid]} but this job uses commit #{job.script_version}"
+        # 'git tag' was successful, or there is an existing tag that points to the same revision.
+        @job_tags[job.uuid] = job.script_version
+      elsif @job_tags[job.uuid] != job.script_version
+        fail_job job, "Existing tag #{job.uuid} points to commit #{@job_tags[job.uuid]} but this job uses commit #{job.script_version}"
+        next
       end
 
       cmd_args << crunch_job_bin
@@ -608,8 +625,11 @@ class Dispatcher
       # fine.
     end
 
-    # Invalidate the per-job auth token
-    j_done[:job_auth].update_attributes expires_at: Time.now
+    # Invalidate the per-job auth token, unless the job is still queued and we
+    # might want to try it again.
+    if jobrecord.state != "Queued"
+      j_done[:job_auth].update_attributes expires_at: Time.now
+    end
 
     @running.delete job_done.uuid
   end