Closes #2923, #3010.
# If this job requires a Docker image, install that.
my $docker_bin = "/usr/bin/docker.io";
-my $docker_image = $Job->{runtime_constraints}->{docker_image} || "";
-if ($docker_image) {
+my ($docker_locator, $docker_hash);
+if ($docker_locator = $Job->{docker_image_locator}) {
+ $docker_hash = find_docker_hash($docker_locator);
+ if (!$docker_hash)
+ {
+ croak("No Docker image hash found from locator $docker_locator");
+ }
+ my $docker_install_script = qq{
+if ! $docker_bin images -q --no-trunc | grep -qxF \Q$docker_hash\E; then
+ arv-get \Q$docker_locator/$docker_hash.tar\E | $docker_bin load
+fi
+};
my $docker_pid = fork();
if ($docker_pid == 0)
{
- srun (["srun", "--nodelist=" . join(' ', @node)],
- [$docker_bin, 'pull', $docker_image]);
+ srun (["srun", "--nodelist=" . join(',', @node)],
+ ["/bin/sh", "-ec", $docker_install_script]);
exit ($?);
}
while (1)
freeze_if_want_freeze ($docker_pid);
select (undef, undef, undef, 0.1);
}
- # If the Docker image was specified as a hash, pull will fail.
- # Ignore that error. We'll see what happens when we try to run later.
- if (($? != 0) && ($docker_image !~ /^[0-9a-fA-F]{5,64}$/))
+ if ($? != 0)
{
- croak("Installing Docker image $docker_image returned exit code $?");
+ croak("Installing Docker image from $docker_locator returned exit code $?");
}
}
"&& perl -";
}
$command .= "&& exec arv-mount --allow-other $ENV{TASK_KEEPMOUNT} --exec ";
- if ($docker_image)
+ if ($docker_hash)
{
$command .= "crunchstat -cgroup-root=/sys/fs/cgroup -cgroup-parent=docker -cgroup-cid=$ENV{TASK_WORK}/docker.cid -poll=10000 ";
$command .= "$docker_bin run -i -a stdin -a stdout -a stderr --cidfile=$ENV{TASK_WORK}/docker.cid ";
$command .= "-e \Q$env_key=$env_val\E ";
}
}
- $command .= "\Q$docker_image\E ";
+ $command .= "\Q$docker_hash\E ";
} else {
$command .= "crunchstat -cgroup-root=/sys/fs/cgroup -poll=10000 "
}
}
}
+sub find_docker_hash {
+ # Given a Keep locator, search for a matching link to find the Docker hash
+ # of the stored image.
+ my $locator = shift;
+ my $links_result = $arv->{links}->{list}->execute(
+ filters => [["head_uuid", "=", $locator],
+ ["link_class", "=", "docker_image_hash"]],
+ limit => 1);
+ my $docker_hash;
+ foreach my $link (@{$links_result->{items}}) {
+ $docker_hash = lc($link->{name});
+ }
+ return $docker_hash;
+}
+
__DATA__
#!/usr/bin/perl
make_link('docker_image_hash', image_hash, **link_base)
if not image_hash.startswith(args.image.lower()):
make_link('docker_image_repository', args.image, **link_base)
- make_link('docker_image_tag', args.tag, **link_base)
+ make_link('docker_image_repo+tag', '{}:{}'.format(args.image, args.tag),
+ **link_base)
# Clean up.
image_file.close()
# Get rid of troublesome nils
users_list.compact!
+ # Load optional keyword arguments, if they exist.
+ if users_list.last.is_a? Hash
+ kwargs = users_list.pop
+ else
+ kwargs = {}
+ end
+
# Check if any of the users are admin. If so, we're done.
if users_list.select { |u| u.is_admin }.empty?
collect { |uuid| sanitize(uuid) }.join(', ')
sql_conds = []
sql_params = []
+ sql_table = kwargs.fetch(:table_name, table_name)
or_object_uuid = ''
# This row is owned by a member of users_list, or owned by a group
# to this row, or to the owner of this row (see join() below).
permitted_uuids = "(SELECT head_uuid FROM links WHERE link_class='permission' AND tail_uuid IN (#{sanitized_uuid_list}))"
- sql_conds += ["#{table_name}.owner_uuid in (?)",
- "#{table_name}.uuid in (?)",
- "#{table_name}.uuid IN #{permitted_uuids}"]
+ sql_conds += ["#{sql_table}.owner_uuid in (?)",
+ "#{sql_table}.uuid in (?)",
+ "#{sql_table}.uuid IN #{permitted_uuids}"]
sql_params += [uuid_list, user_uuids]
- if self == Link and users_list.any?
+ if sql_table == "links" and users_list.any?
# This row is a 'permission' or 'resources' link class
# The uuid for a member of users_list is referenced in either the head
# or tail of the link
- sql_conds += ["(#{table_name}.link_class in (#{sanitize 'permission'}, #{sanitize 'resources'}) AND (#{table_name}.head_uuid IN (?) OR #{table_name}.tail_uuid IN (?)))"]
+ sql_conds += ["(#{sql_table}.link_class in (#{sanitize 'permission'}, #{sanitize 'resources'}) AND (#{sql_table}.head_uuid IN (?) OR #{sql_table}.tail_uuid IN (?)))"]
sql_params += [user_uuids, user_uuids]
end
- if self == Log and users_list.any?
+ if sql_table == "logs" and users_list.any?
# Link head points to the object described by this row
- sql_conds += ["#{table_name}.object_uuid IN #{permitted_uuids}"]
+ sql_conds += ["#{sql_table}.object_uuid IN #{permitted_uuids}"]
# This object described by this row is owned by this user, or owned by a group readable by this user
- sql_conds += ["#{table_name}.object_owner_uuid in (?)"]
+ sql_conds += ["#{sql_table}.object_owner_uuid in (?)"]
sql_params += [uuid_list]
end
raise "uuid #{uuid} has no hash part" if !hash_part
[hash_part, size_part].compact.join '+'
end
+
+ def self.for_latest_docker_image(search_term, search_tag=nil, readers=nil)
+ readers ||= [Thread.current[:user]]
+ base_search = Link.
+ readable_by(*readers).
+ readable_by(*readers, table_name: "collections").
+ joins("JOIN collections ON links.head_uuid = collections.uuid").
+ order("links.created_at DESC")
+
+ # If the search term is a Collection locator with an associated
+ # Docker image hash link, return that Collection.
+ coll_matches = base_search.
+ where(link_class: "docker_image_hash", collections: {uuid: search_term})
+ if match = coll_matches.first
+ return find_by_uuid(match.head_uuid)
+ end
+
+ # Find Collections with matching Docker image repository+tag pairs.
+ matches = base_search.
+ where(link_class: "docker_image_repo+tag",
+ name: "#{search_term}:#{search_tag || 'latest'}")
+
+ # If that didn't work, find Collections with matching Docker image hashes.
+ if matches.empty?
+ matches = base_search.
+ where("link_class = ? and name LIKE ?",
+ "docker_image_hash", "#{search_term}%")
+ end
+
+ # Select the image that was created most recently. Note that the
+ # SQL search order and fallback timestamp values are chosen so
+ # that if image timestamps are missing, we use the image with the
+ # newest link.
+ latest_image_link = nil
+ latest_image_timestamp = "1900-01-01T00:00:00Z"
+ matches.find_each do |link|
+ link_timestamp = link.properties.fetch("image_timestamp",
+ "1900-01-01T00:00:01Z")
+ if link_timestamp > latest_image_timestamp
+ latest_image_link = link
+ latest_image_timestamp = link_timestamp
+ end
+ end
+ latest_image_link.nil? ? nil : find_by_uuid(latest_image_link.head_uuid)
+ end
end
include HasUuid
include KindAndEtag
include CommonApiTemplate
+ attr_protected :docker_image_locator
serialize :script_parameters, Hash
serialize :runtime_constraints, Hash
serialize :tasks_summary, Hash
+ before_validation :find_docker_image_locator
before_create :ensure_unique_submit_id
before_create :ensure_script_version_is_commit
before_update :ensure_script_version_is_commit
t.add :nondeterministic
t.add :repository
t.add :supplied_script_version
+ t.add :docker_image_locator
end
def assert_finished
true
end
+ def find_docker_image_locator
+ # Find the Collection that holds the Docker image specified in the
+ # runtime constraints, and store its locator in docker_image_locator.
+ image_search = runtime_constraints['docker_image']
+ image_tag = runtime_constraints['docker_image_tag']
+ if image_search.nil?
+ self.docker_image_locator = nil
+ elsif coll = Collection.for_latest_docker_image(image_search, image_tag)
+ self.docker_image_locator = coll.uuid
+ else
+ errors.add(:docker_image_locator, "Docker image not found")
+ false
+ end
+ end
+
def dependencies
deps = {}
queue = self.script_parameters.values
--- /dev/null
+class AddDockerLocatorToJobs < ActiveRecord::Migration
+ def change
+ add_column :jobs, :docker_image_locator, :string
+ end
+end
#
# It's strongly recommended to check this file into your version control system.
+ActiveRecord::Schema.define(:version => 20140611173003) do
+
-ActiveRecord::Schema.define(:version => 20140602143352) do
create_table "api_client_authorizations", :force => true do |t|
t.string "api_token", :null => false
t.string "repository"
t.boolean "output_is_persistent", :default => false, :null => false
t.string "supplied_script_version"
+ t.string "docker_image_locator"
end
add_index "jobs", ["created_at"], :name => "index_jobs_on_created_at"
modified_at: 2014-02-03T17:22:54Z
updated_at: 2014-02-03T17:22:54Z
manifest_text: "./dir1/sub1 0:0:a 0:0:b\n./dir2/sub2 0:0:c 0:0:d\n"
+
+docker_image:
+ # This Collection has links with Docker image metadata.
+ uuid: fa3c1a9cb6783f85f2ecda037e07b8c3+167
+ owner_uuid: qr1hi-tpzed-000000000000000
+ created_at: 2014-06-11T17:22:54Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2014-06-11T17:22:54Z
+ updated_at: 2014-06-11T17:22:54Z
+ manifest_text: ". d21353cfe035e3e384563ee55eadbb2f+67108864 5c77a43e329b9838cbec18ff42790e57+55605760 0:122714624:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678.tar\n"
uuid: zzzzz-o0j2j-enl1wg58310loc6
owner_uuid: zzzzz-tpzed-000000000000000
created_at: 2014-05-28 16:24:02.314722162 Z
- modified_by_client_uuid:
+ modified_by_client_uuid:
modified_by_user_uuid: zzzzz-tpzed-000000000000000
modified_at: 2014-05-28 16:24:02.314484982 Z
tail_uuid: ~
tail_uuid: ~
head_uuid: ~
properties: {}
+
+active_user_permission_to_docker_image_collection:
+ uuid: zzzzz-o0j2j-dp1d8395ldqw33s
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2014-01-24 20:42:26 -0800
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2014-01-24 20:42:26 -0800
+ updated_at: 2014-01-24 20:42:26 -0800
+ tail_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ link_class: permission
+ name: can_read
+ head_uuid: fa3c1a9cb6783f85f2ecda037e07b8c3+167
+ properties: {}
+
+docker_image_collection_hash:
+ uuid: zzzzz-o0j2j-dockercollhasha
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2014-06-11 14:30:00.184389725 Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2014-06-11 14:30:00.184019565 Z
+ updated_at: 2014-06-11 14:30:00.183829316 Z
+ link_class: docker_image_hash
+ name: d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678
+ tail_uuid: ~
+ head_uuid: fa3c1a9cb6783f85f2ecda037e07b8c3+167
+ properties:
+ image_timestamp: 2014-06-10T14:30:00.184019565Z
+
+docker_image_collection_repository:
+ uuid: zzzzz-o0j2j-dockercollrepos
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2014-06-11 14:30:00.184389725 Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2014-06-11 14:30:00.184019565 Z
+ updated_at: 2014-06-11 14:30:00.183829316 Z
+ link_class: docker_image_repository
+ name: arvados/apitestfixture
+ tail_uuid: ~
+ head_uuid: fa3c1a9cb6783f85f2ecda037e07b8c3+167
+ properties:
+ image_timestamp: 2014-06-10T14:30:00.184019565Z
+
+docker_image_collection_tag:
+ uuid: zzzzz-o0j2j-dockercolltagbb
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2014-06-11 14:30:00.184389725 Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2014-06-11 14:30:00.184019565 Z
+ updated_at: 2014-06-11 14:30:00.183829316 Z
+ link_class: docker_image_repo+tag
+ name: arvados/apitestfixture:latest
+ tail_uuid: ~
+ head_uuid: fa3c1a9cb6783f85f2ecda037e07b8c3+167
+ properties:
+ image_timestamp: 2014-06-10T14:30:00.184019565Z
+
+docker_image_collection_tag2:
+ uuid: zzzzz-o0j2j-dockercolltagbc
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2014-06-11 14:30:00.184389725 Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2014-06-11 14:30:00.184019565 Z
+ updated_at: 2014-06-11 14:30:00.183829316 Z
+ link_class: docker_image_repo+tag
+ name: arvados/apitestfixture:june10
+ tail_uuid: ~
+ head_uuid: fa3c1a9cb6783f85f2ecda037e07b8c3+167
+ properties:
+ image_timestamp: 2014-06-10T14:30:00.184019565Z
+
+ancient_docker_image_collection_hash:
+ # This image helps test that searches for Docker images find
+ # the latest available image: the hash is the same as
+ # docker_image_collection_hash, but it points to a different
+ # Collection and has an older image timestamp.
+ uuid: zzzzz-o0j2j-dockercollhashz
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2014-06-12 14:30:00.184389725 Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2014-06-12 14:30:00.184019565 Z
+ updated_at: 2014-06-12 14:30:00.183829316 Z
+ link_class: docker_image_hash
+ name: d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678
+ tail_uuid: ~
+ head_uuid: b519d9cb706a29fc7ea24dbea2f05851+249025
+ properties:
+ image_timestamp: 2010-06-10T14:30:00.184019565Z
end
end
+ def assert_link_classes_ascend(current_class, prev_class)
+ # Databases and Ruby don't always agree about string ordering with
+ # punctuation. If the strings aren't ascending normally, check
+ # that they're equal up to punctuation.
+ if current_class < prev_class
+ class_prefix = current_class.split(/\W/).first
+ assert prev_class.start_with?(class_prefix)
+ end
+ end
+
test "select two columns with order" do
get "/arvados/v1/links", {:format => :json, :select => ['link_class', 'uuid'], :order => ['link_class asc', "uuid desc"]}, auth(:active)
assert_response :success
prev_uuid = "zzzzz-zzzzz-zzzzzzzzzzzzzzz"
end
- assert i['link_class'] >= prev_link_class
+ assert_link_classes_ascend(i['link_class'], prev_link_class)
assert i['uuid'] < prev_uuid
prev_link_class = i['link_class']
prev_uuid = "zzzzz-zzzzz-zzzzzzzzzzzzzzz"
end
- assert i['link_class'] >= prev_link_class
+ assert_link_classes_ascend(i['link_class'], prev_link_class)
assert i['uuid'] < prev_uuid
prev_link_class = i['link_class']
job: {
repository: 'foo',
- runtime_constraints: {docker_image: 'arvados/jobs'},
+ runtime_constraints: {docker_image: 'arvados/apitestfixture'},
script: 'hash',
script_version: 'master',
script_parameters: {pattern: 'foobar'},
require 'test_helper'
class JobTest < ActiveSupport::TestCase
- # test "the truth" do
- # assert true
- # end
+ BAD_COLLECTION = "#{'f' * 32}+0"
+
+ setup do
+ set_user_from_auth :active
+ end
+
+ test "Job without Docker image doesn't get locator" do
+ job = Job.new
+ assert job.valid?
+ assert_nil job.docker_image_locator
+ end
+
+ { 'name' => [:links, :docker_image_collection_repository, :name],
+ 'hash' => [:links, :docker_image_collection_hash, :name],
+ 'locator' => [:collections, :docker_image, :uuid],
+ }.each_pair do |spec_type, (fixture_type, fixture_name, fixture_attr)|
+ test "Job initialized with Docker image #{spec_type} gets locator" do
+ image_spec = send(fixture_type, fixture_name).send(fixture_attr)
+ job = Job.new(runtime_constraints: {'docker_image' => image_spec})
+ assert(job.valid?, "Docker image #{spec_type} was invalid")
+ assert_equal(collections(:docker_image).uuid, job.docker_image_locator)
+ end
+
+ test "Job modified with Docker image #{spec_type} gets locator" do
+ job = Job.new
+ assert job.valid?
+ assert_nil job.docker_image_locator
+ image_spec = send(fixture_type, fixture_name).send(fixture_attr)
+ job.runtime_constraints['docker_image'] = image_spec
+ assert(job.valid?, "modified Docker image #{spec_type} was invalid")
+ assert_equal(collections(:docker_image).uuid, job.docker_image_locator)
+ end
+ end
+
+ test "removing a Docker runtime constraint removes the locator" do
+ image_locator = collections(:docker_image).uuid
+ job = Job.new(runtime_constraints: {'docker_image' => image_locator})
+ assert job.valid?
+ assert_equal(image_locator, job.docker_image_locator)
+ job.runtime_constraints = {}
+ assert(job.valid?, "clearing runtime constraints made the Job invalid")
+ assert_nil job.docker_image_locator
+ end
+
+ test "locate a Docker image with a repository + tag" do
+ image_repo, image_tag =
+ links(:docker_image_collection_tag2).name.split(':', 2)
+ job = Job.new(runtime_constraints:
+ {'docker_image' => image_repo,
+ 'docker_image_tag' => image_tag})
+ assert(job.valid?, "Job with Docker tag search invalid")
+ assert_equal(collections(:docker_image).uuid, job.docker_image_locator)
+ end
+
+ test "can't locate a Docker image with a nonexistent tag" do
+ image_repo = links(:docker_image_collection_repository).name
+ image_tag = '__nonexistent tag__'
+ job = Job.new(runtime_constraints:
+ {'docker_image' => image_repo,
+ 'docker_image_tag' => image_tag})
+ assert(job.invalid?, "Job with bad Docker tag valid")
+ end
+
+ test "locate a Docker image with a partial hash" do
+ image_hash = links(:docker_image_collection_hash).name[0..24]
+ job = Job.new(runtime_constraints: {'docker_image' => image_hash})
+ assert(job.valid?, "Job with partial Docker image hash failed")
+ assert_equal(collections(:docker_image).uuid, job.docker_image_locator)
+ end
+
+ { 'name' => 'arvados_test_nonexistent',
+ 'hash' => 'f' * 64,
+ 'locator' => BAD_COLLECTION,
+ }.each_pair do |spec_type, image_spec|
+ test "Job validation fails with nonexistent Docker image #{spec_type}" do
+ job = Job.new(runtime_constraints: {'docker_image' => image_spec})
+ assert(job.invalid?, "nonexistent Docker image #{spec_type} was valid")
+ end
+ end
+
+ test "Job validation fails with non-Docker Collection constraint" do
+ job = Job.new(runtime_constraints:
+ {'docker_image' => collections(:foo_file).uuid})
+ assert(job.invalid?, "non-Docker Collection constraint was valid")
+ end
+
+ test "can't create Job with Docker image locator" do
+ begin
+ job = Job.new(docker_image_locator: BAD_COLLECTION)
+ rescue ActiveModel::MassAssignmentSecurity::Error
+ # Test passes - expected attribute protection
+ else
+ assert_nil job.docker_image_locator
+ end
+ end
+
+ test "can't assign Docker image locator to Job" do
+ job = Job.new
+ begin
+ Job.docker_image_locator = BAD_COLLECTION
+ rescue NoMethodError
+ # Test passes - expected attribute protection
+ end
+ assert_nil job.docker_image_locator
+ end
end
projects / arvados.git / commitdiff