From 526f0fe659be1d21f0f30aba95f643d690122ded Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Mon, 28 Jun 2021 15:58:57 -0400 Subject: [PATCH] 17395: Add output_storage_classes to container and container_request Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- sdk/go/arvados/container.go | 2 + services/api/app/models/container.rb | 8 ++- services/api/app/models/container_request.rb | 13 ++++- services/api/db/structure.sql | 55 ++++++++++--------- .../api/test/unit/container_request_test.rb | 25 +++++++++ 5 files changed, 72 insertions(+), 31 deletions(-) diff --git a/sdk/go/arvados/container.go b/sdk/go/arvados/container.go index f542634708..b57dc84944 100644 --- a/sdk/go/arvados/container.go +++ b/sdk/go/arvados/container.go @@ -32,6 +32,7 @@ type Container struct { FinishedAt *time.Time `json:"finished_at"` // nil if not yet finished GatewayAddress string `json:"gateway_address"` InteractiveSessionStarted bool `json:"interactive_session_started"` + OutputStorageClasses []string `json:"output_storage_classes"` } // ContainerRequest is an arvados#container_request resource. @@ -69,6 +70,7 @@ type ContainerRequest struct { ExpiresAt time.Time `json:"expires_at"` Filters []Filter `json:"filters"` ContainerCount int `json:"container_count"` + OutputStorageClasses []string `json:"output_storage_classes"` } // Mount is special behavior to attach to a filesystem path or device. diff --git a/services/api/app/models/container.rb b/services/api/app/models/container.rb index e6d945a005..a844fbaf77 100644 --- a/services/api/app/models/container.rb +++ b/services/api/app/models/container.rb @@ -22,6 +22,7 @@ class Container < ArvadosModel attribute :secret_mounts, :jsonbHash, default: {} attribute :runtime_status, :jsonbHash, default: {} attribute :runtime_auth_scopes, :jsonbHash, default: {} + attribute :output_storage_classes, :jsonbArray, default: ["default"] serialize :environment, Hash serialize :mounts, Hash @@ -79,6 +80,7 @@ class Container < ArvadosModel t.add :lock_count t.add :gateway_address t.add :interactive_session_started + t.add :output_storage_classes end # Supported states for a container @@ -187,7 +189,8 @@ class Container < ArvadosModel secret_mounts: req.secret_mounts, runtime_token: req.runtime_token, runtime_user_uuid: runtime_user.uuid, - runtime_auth_scopes: runtime_auth_scopes + runtime_auth_scopes: runtime_auth_scopes, + output_storage_classes: req.output_storage_classes, } end act_as_system_user do @@ -467,7 +470,8 @@ class Container < ArvadosModel :environment, :mounts, :output_path, :priority, :runtime_constraints, :scheduling_parameters, :secret_mounts, :runtime_token, - :runtime_user_uuid, :runtime_auth_scopes) + :runtime_user_uuid, :runtime_auth_scopes, + :output_storage_classes) end case self.state diff --git a/services/api/app/models/container_request.rb b/services/api/app/models/container_request.rb index e712acc6e9..f47fff792f 100644 --- a/services/api/app/models/container_request.rb +++ b/services/api/app/models/container_request.rb @@ -23,6 +23,7 @@ class ContainerRequest < ArvadosModel # already know how to properly treat them. attribute :properties, :jsonbHash, default: {} attribute :secret_mounts, :jsonbHash, default: {} + attribute :output_storage_classes, :jsonbArray, default: ["default"] serialize :environment, Hash serialize :mounts, Hash @@ -76,6 +77,7 @@ class ContainerRequest < ArvadosModel t.add :scheduling_parameters t.add :state t.add :use_existing + t.add :output_storage_classes end # Supported states for a container request @@ -97,7 +99,8 @@ class ContainerRequest < ArvadosModel :container_image, :cwd, :environment, :filters, :mounts, :output_path, :priority, :runtime_token, :runtime_constraints, :state, :container_uuid, :use_existing, - :scheduling_parameters, :secret_mounts, :output_name, :output_ttl] + :scheduling_parameters, :secret_mounts, :output_name, :output_ttl, + :output_storage_classes] def self.limit_index_columns_read ["mounts"] @@ -177,7 +180,9 @@ class ContainerRequest < ArvadosModel 'container_uuid' => container_uuid, }, portable_data_hash: log_col.portable_data_hash, - manifest_text: log_col.manifest_text) + manifest_text: log_col.manifest_text, + storage_classes_desired: self.output_storage_classes + ) completed_coll.save_with_unique_name! end end @@ -211,6 +216,7 @@ class ContainerRequest < ArvadosModel owner_uuid: self.owner_uuid, name: coll_name, manifest_text: "", + storage_classes_desired: self.output_storage_classes, properties: { 'type' => out_type, 'container_request' => uuid, @@ -296,7 +302,8 @@ class ContainerRequest < ArvadosModel log_coll = Collection.new( owner_uuid: self.owner_uuid, name: coll_name = "Container log for request #{uuid}", - manifest_text: "") + manifest_text: "", + storage_classes_desired: self.output_storage_classes) end # copy logs from old container into CR's log collection diff --git a/services/api/db/structure.sql b/services/api/db/structure.sql index 14eca609eb..2bca887212 100644 --- a/services/api/db/structure.sql +++ b/services/api/db/structure.sql @@ -238,6 +238,29 @@ SET default_tablespace = ''; SET default_with_oids = false; +-- +-- Name: groups; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.groups ( + id integer NOT NULL, + uuid character varying(255), + owner_uuid character varying(255), + created_at timestamp without time zone NOT NULL, + modified_by_client_uuid character varying(255), + modified_by_user_uuid character varying(255), + modified_at timestamp without time zone, + name character varying(255) NOT NULL, + description character varying(524288), + updated_at timestamp without time zone NOT NULL, + group_class character varying(255), + trash_at timestamp without time zone, + is_trashed boolean DEFAULT false NOT NULL, + delete_at timestamp without time zone, + properties jsonb DEFAULT '{}'::jsonb +); + + -- -- Name: api_client_authorizations; Type: TABLE; Schema: public; Owner: - -- @@ -461,7 +484,8 @@ CREATE TABLE public.container_requests ( output_name character varying(255) DEFAULT NULL::character varying, output_ttl integer DEFAULT 0 NOT NULL, secret_mounts jsonb DEFAULT '{}'::jsonb, - runtime_token text + runtime_token text, + output_storage_classes jsonb DEFAULT '["default"]'::jsonb ); @@ -523,7 +547,8 @@ CREATE TABLE public.containers ( runtime_token text, lock_count integer DEFAULT 0 NOT NULL, gateway_address character varying, - interactive_session_started boolean DEFAULT false NOT NULL + interactive_session_started boolean DEFAULT false NOT NULL, + output_storage_classes jsonb DEFAULT '["default"]'::jsonb ); @@ -546,29 +571,6 @@ CREATE SEQUENCE public.containers_id_seq ALTER SEQUENCE public.containers_id_seq OWNED BY public.containers.id; --- --- Name: groups; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.groups ( - id integer NOT NULL, - uuid character varying(255), - owner_uuid character varying(255), - created_at timestamp without time zone NOT NULL, - modified_by_client_uuid character varying(255), - modified_by_user_uuid character varying(255), - modified_at timestamp without time zone, - name character varying(255) NOT NULL, - description character varying(524288), - updated_at timestamp without time zone NOT NULL, - group_class character varying(255), - trash_at timestamp without time zone, - is_trashed boolean DEFAULT false NOT NULL, - delete_at timestamp without time zone, - properties jsonb DEFAULT '{}'::jsonb -); - - -- -- Name: groups_id_seq; Type: SEQUENCE; Schema: public; Owner: - -- @@ -3191,6 +3193,7 @@ INSERT INTO "schema_migrations" (version) VALUES ('20201105190435'), ('20201202174753'), ('20210108033940'), -('20210126183521'); +('20210126183521'), +('20210621204455'); diff --git a/services/api/test/unit/container_request_test.rb b/services/api/test/unit/container_request_test.rb index 2d5c735181..f28e38faae 100644 --- a/services/api/test/unit/container_request_test.rb +++ b/services/api/test/unit/container_request_test.rb @@ -1290,4 +1290,29 @@ class ContainerRequestTest < ActiveSupport::TestCase cr.save! end end + + test "default output_storage_classes" do + act_as_user users(:active) do + cr = create_minimal_req!(priority: 1, + state: ContainerRequest::Committed, + output_name: 'foo') + run_container(cr) + cr.reload + output = Collection.find_by_uuid(cr.output_uuid) + assert_equal ["default"], output.storage_classes_desired + end + end + + test "setting output_storage_classes" do + act_as_user users(:active) do + cr = create_minimal_req!(priority: 1, + state: ContainerRequest::Committed, + output_name: 'foo', + output_storage_classes: ["foo_storage_class", "bar_storage_class"]) + run_container(cr) + cr.reload + output = Collection.find_by_uuid(cr.output_uuid) + assert_equal ["foo_storage_class", "bar_storage_class"], output.storage_classes_desired + end + end end -- 2.30.2