def find_objects_for_index
@objects ||= model_class
- @objects = @objects.filter(@filters).limit(@limit).offset(@offset)
+ @objects = @objects.filter(@filters).limit(@limit).offset(@offset).order(@order)
@objects.fetch_multiple_pages(false)
end
ConfigValidators.validate_wb2_url_config()
ConfigValidators.validate_download_config()
end
+ if Rails.configuration.Users.AnonymousUserToken and
+ !Rails.configuration.Users.AnonymousUserToken.starts_with?("v2/")
+ Rails.configuration.Users.AnonymousUserToken = "v2/#{clusterID}-gj3su-anonymouspublic/#{Rails.configuration.Users.AnonymousUserToken}"
+ end
end
# Usage: handle_libarvados_perl
handle_libarvados_perl () {
- if [[ -n "$ONLY_BUILD" ]] || [[ "$ONLY_BUILD" != "libarvados-perl" ]] ; then
+ if [[ -n "$ONLY_BUILD" ]] && [[ "$ONLY_BUILD" != "libarvados-perl" ]] ; then
debug_echo -e "Skipping build of libarvados-perl package."
return 0
fi
- cd "$WORKSPACE/sdk/perl"
+ # The perl sdk subdirectory is so old that it has no tag in its history,
+ # which causes version_at_commit.sh to fail. Just rebuild it every time.
+ cd "$WORKSPACE"
libarvados_perl_version="$(version_from_git)"
+ cd "$WORKSPACE/sdk/perl"
cd $WORKSPACE/packages/$TARGET
test_package_presence libarvados-perl "$libarvados_perl_version"
perl Makefile.PL INSTALL_BASE=install >"$STDOUT_IF_DEBUG" && \
make install INSTALLDIRS=perl >"$STDOUT_IF_DEBUG" && \
fpm_build "$WORKSPACE/sdk/perl" install/lib/=/usr/share libarvados-perl \
- dir "$(version_from_git)" install/man/=/usr/share/man \
+ dir "$libarvados_perl_version" install/man/=/usr/share/man \
"$WORKSPACE/apache-2.0.txt=/usr/share/doc/libarvados-perl/apache-2.0.txt" && \
mv --no-clobber libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/"
fi
done
"$bundle" version | tee /dev/stderr | grep -q 'version 2'
) || fatal 'install bundler'
+ if test -d /var/lib/arvados-arvbox/ ; then
+ # Inside arvbox, use bundler-installed binstubs. The
+ # system bundler and rail's own bin/bundle refuse to work.
+ # I don't know why.
+ bundle=binstubs/bundle
+ fi
fi
}
<div class="releasenotes">
</notextile>
-h2(#main). development main (as of 2022-03-??)
+h2(#main). development main (as of 2022-04-08)
-h2(#v2_4_0). v2.4.0 (2022-03-??)
+"previous: Upgrading to 2.4.0":#v2_4_0
+
+h2(#v2_4_0). v2.4.0 (2022-04-08)
"previous: Upgrading to 2.3.1":#v2_3_1
+h3. Default result order changed
+
+When requesting a list of objects without an explicit @order@ parameter, the default order has changed from @modified_at desc, uuid asc@ to @modified_at desc, uuid desc@. This means that if two objects have identical @modified_at@ timestamps, the tiebreaker will now be based on @uuid@ in decending order where previously it would be ascending order. The practical effect of this should be minor; with microsecond precision it is unusual to have two records with exactly the same timestamp, and order-sensitive queries should already provide an explicit @order@ parameter.
+
h3. Ubuntu 18.04 Arvados Python packages now depend on python-3.8
Ubuntu 18.04 ships with Python 3.6 as the default version of Python 3. Ubuntu also ships a version of Python 3.8, and the Arvados Python packages (@python3-arvados-cwl-runner@, @python3-arvados-fuse@, @python3-arvados-python-client@, @python3-arvados-user-activity@ and @python3-crunchstat-summary@) now depend on the @python-3.8@ system package.
h3. Anonymous token changes
-The anonymous token configured in @Users.AnonymousUserToken@ must now be 32 characters or longer. This was already the suggestion in the documentation, now it is enforced. The @script/get_anonymous_user_token.rb@ script that was needed to register the anonymous user token in the database has been removed. Registration of the anonymous token is no longer necessary. If the anonymous token in @config.yml@ is specified as a full V2 token, that will now generate a warning - it should be updated to list just the secret (i.e. the part after the last forward slash).
+The anonymous token configured in @Users.AnonymousUserToken@ must now be 32 characters or longer. This was already the suggestion in the documentation, now it is enforced. The @script/get_anonymous_user_token.rb@ script that was needed to register the anonymous user token in the database has been removed. Registration of the anonymous token is no longer necessary.
h3. Preemptible instance support changes
cwltool:CUDARequirement:
cudaVersionMin: "11.0"
- cudaComputeCapabilityMin: "9.0"
- deviceCountMin: 1
- deviceCountMax: 1
+ cudaComputeCapability: "9.0"
+ cudaDeviceCountMin: 1
+ cudaDeviceCountMax: 1
arv:UsePreemptible:
usePreemptible: true
table(table table-bordered table-condensed).
|_. Field |_. Type |_. Description |
|cudaVersionMin|string|Required. The CUDA SDK version corresponding to the minimum driver version supported by the container (generally, the SDK version 'X.Y' the application was compiled against).|
-|cudaComputeCapabilityMin|string|Required. The minimum CUDA hardware capability (in 'X.Y' format) required by the application's PTX or C++ GPU code (will be JIT compiled for the available hardware).|
-|deviceCountMin|integer|Minimum number of GPU devices to allocate on a single node. Required.|
-|deviceCountMax|integer|Maximum number of GPU devices to allocate on a single node. Optional. If not specified, same as @minDeviceCount@.|
+|cudaComputeCapability|string|Required. The minimum CUDA hardware capability (in 'X.Y' format) required by the application's PTX or C++ GPU code (will be JIT compiled for the available hardware).|
+|cudaDeviceCountMin|integer|Minimum number of GPU devices to allocate on a single node. Required.|
+|cudaDeviceCountMax|integer|Maximum number of GPU devices to allocate on a single node. Optional. If not specified, same as @cudaDeviceCountMin@.|
h2(#UsePreemptible). arv:UsePreemptible
if !strings.HasPrefix(token, "v2/") {
return fmt.Errorf("%s: unacceptable characters in token (only a-z, A-Z, 0-9 are acceptable)", label)
}
- ldr.Logger.Warnf("%s: token is a full V2 token, should just be a secret (remove everything up to and including the last forward slash)", label)
if !acceptableTokenRe.MatchString(tmp[2]) {
return fmt.Errorf("%s: unacceptable characters in V2 token secret (only a-z, A-Z, 0-9 are acceptable)", label)
}
+ if len(tmp[2]) < acceptableTokenLength {
+ ldr.Logger.Warnf("%s: secret is too short (should be at least %d characters)", label, acceptableTokenLength)
+ }
} else if len(token) < acceptableTokenLength {
if ldr.Logger != nil {
ldr.Logger.Warnf("%s: token is too short (should be at least %d characters)", label, acceptableTokenLength)
return nil, errors.New("no token provided")
}
for _, token := range incoming.Tokens {
- if strings.HasPrefix(token, "v2/"+cluster.ClusterID+"-") && remoteID == cluster.Login.LoginCluster {
- // If we did this, the login cluster
- // would call back to us and then
- // reject our response because the
- // user UUID prefix (i.e., the
- // LoginCluster prefix) won't match
- // the token UUID prefix (i.e., our
- // prefix).
+ if strings.HasPrefix(token, "v2/"+cluster.ClusterID+"-") &&
+ !strings.HasPrefix(token, "v2/"+cluster.ClusterID+"-gj3su-anonymouspublic/") &&
+ remoteID == cluster.Login.LoginCluster {
+ // If we did this, the login cluster would call back to us and then
+ // reject our response because the user UUID prefix (i.e., the
+ // LoginCluster prefix) won't match the token UUID prefix (i.e., our
+ // prefix). The anonymous token is OK to forward, because (unlike other
+ // local tokens for real users) the validation callback will return the
+ // locally issued anonymous user ID instead of a login-cluster user ID.
+ // That anonymous user ID gets mapped to the local anonymous user
+ // automatically on the login cluster.
return nil, httpErrorf(http.StatusUnauthorized, "cannot use a locally issued token to forward a request to our login cluster (%s)", remoteID)
}
salted, err := auth.SaltToken(token, remoteID)
c.Check(coll.PortableDataHash, check.Equals, pdh)
}
+// z3333 should forward the locally-issued anonymous user token to its login
+// cluster z1111. That is no problem because the login cluster controller will
+// map any anonymous user token to its local anonymous user.
+//
+// This needs to work because wb1 has a tendency to slap the local anonymous
+// user token on every request as a reader_token, which gets folded into the
+// request token list controller.
+//
+// Use a z1111 user token and the anonymous token from z3333 passed in as a
+// reader_token to do a request on z3333, asking for the z1111 anonymous user
+// object. The request will be forwarded to the z1111 cluster. The presence of
+// the z3333 anonymous user token should not prohibit the request from being
+// forwarded.
+func (s *IntegrationSuite) TestForwardAnonymousTokenToLoginCluster(c *check.C) {
+ conn1 := s.testClusters["z1111"].Conn()
+ s.testClusters["z3333"].Conn()
+
+ rootctx1, _, _ := s.testClusters["z1111"].RootClients()
+ _, anonac3, _ := s.testClusters["z3333"].AnonymousClients()
+
+ // Make a user connection to z3333 (using a z1111 user, because that's the login cluster)
+ _, userac1, _, _ := s.testClusters["z3333"].UserClients(rootctx1, c, conn1, "user@example.com", true)
+
+ // Get the anonymous user token for z3333
+ var anon3Auth arvados.APIClientAuthorization
+ err := anonac3.RequestAndDecode(&anon3Auth, "GET", "/arvados/v1/api_client_authorizations/current", nil, nil)
+ c.Check(err, check.IsNil)
+
+ var userList arvados.UserList
+ where := make(map[string]string)
+ where["uuid"] = "z1111-tpzed-anonymouspublic"
+ err = userac1.RequestAndDecode(&userList, "GET", "/arvados/v1/users", nil,
+ map[string]interface{}{
+ "reader_tokens": []string{anon3Auth.TokenV2()},
+ "where": where,
+ },
+ )
+ // The local z3333 anonymous token must be allowed to be forwarded to the login cluster
+ c.Check(err, check.IsNil)
+
+ userac1.AuthToken = "v2/z1111-gj3su-asdfasdfasdfasd/this-token-does-not-validate-so-anonymous-token-will-be-used-instead"
+ err = userac1.RequestAndDecode(&userList, "GET", "/arvados/v1/users", nil,
+ map[string]interface{}{
+ "reader_tokens": []string{anon3Auth.TokenV2()},
+ "where": where,
+ },
+ )
+ c.Check(err, check.IsNil)
+}
+
// Get a token from the login cluster (z1111), use it to submit a
// container request on z2222.
func (s *IntegrationSuite) TestCreateContainerRequestWithFedToken(c *check.C) {
Package: ArvadosR
Type: Package
Title: Arvados R SDK
-Version: 0.0.6
+Version: 2.4.0
Authors@R: c(person("Fuad", "Muhic", role = c("aut", "ctr"), email = "fmuhic@capeannenterprises.com"),
person("Peter", "Amstutz", role = c("cre"), email = "peter.amstutz@curii.com"))
Description: This is the Arvados R SDK
pagesize = 1000
kwargs["limit"] = pagesize
kwargs["count"] = 'none'
- kwargs["order"] = ["%s %s" % (order_key, "asc" if ascending else "desc"), "uuid asc"]
+ asc = "asc" if ascending else "desc"
+ kwargs["order"] = ["%s %s" % (order_key, asc), "uuid %s" % asc]
other_filters = kwargs.get("filters", [])
if "select" in kwargs and "uuid" not in kwargs["select"]:
if firstitem[order_key] == lastitem[order_key]:
# Got a page where every item has the same order key.
# Switch to using uuid for paging.
- nextpage = [[order_key, "=", lastitem[order_key]], ["uuid", ">", lastitem["uuid"]]]
+ nextpage = [[order_key, "=", lastitem[order_key]], ["uuid", ">" if ascending else "<", lastitem["uuid"]]]
prev_page_all_same_order_key = True
else:
# Start from the last order key seen, but skip the last
def test_onepage_desc(self):
ks = KeysetTestHelper([[
- {"limit": 1000, "count": "none", "order": ["created_at desc", "uuid asc"], "filters": []},
+ {"limit": 1000, "count": "none", "order": ["created_at desc", "uuid desc"], "filters": []},
{"items": [{"created_at": "2", "uuid": "2"}, {"created_at": "1", "uuid": "1"}]}
], [
- {"limit": 1000, "count": "none", "order": ["created_at desc", "uuid asc"], "filters": [["created_at", "<=", "1"], ["uuid", "!=", "1"]]},
+ {"limit": 1000, "count": "none", "order": ["created_at desc", "uuid desc"], "filters": [["created_at", "<=", "1"], ["uuid", "!=", "1"]]},
{"items": []}
]])
def get_permissions
if current_user.andand.can?(manage: @object)
# find all links and return them
- @objects = Link.where(link_class: "permission",
- head_uuid: params[:uuid])
+ @objects = Link.unscoped.where(link_class: "permission",
+ head_uuid: params[:uuid])
@offset = 0
@limit = @objects.count
render_list
protected
def find_object_by_uuid
+ if params[:id] && params[:id].match(/\D/)
+ params[:uuid] = params.delete :id
+ end
if action_name == 'get_permissions'
# get_permissions accepts a UUID for any kind of object.
@object = ArvadosModel::resource_class_for_uuid(params[:uuid])
.readable_by(*@read_users)
.where(uuid: params[:uuid])
.first
- else
+ elsif !current_user
super
+ else
+ # The usual permission-filtering index query is unnecessarily
+ # inefficient, and doesn't match all permission links that
+ # should be visible (see #18865). Instead, we look up the link
+ # by UUID, then check whether (a) its tail_uuid is the current
+ # user or (b) its head_uuid is an object the current_user
+ # can_manage.
+ link = Link.unscoped.where(uuid: params[:uuid]).first
+ if link && link.link_class != 'permission'
+ # Not a permission link. Re-fetch using generic
+ # permission-filtering query.
+ super
+ elsif link && (current_user.uuid == link.tail_uuid ||
+ current_user.can?(manage: link.head_uuid))
+ # Permission granted.
+ @object = link
+ else
+ # Permission denied, i.e., link is invisible => 404.
+ @object = nil
+ end
end
end
k
end
end
+
+ # If the provided filters are enough to limit the results to
+ # permission links with specific head_uuids or
+ # tail_uuid=current_user, bypass the normal readable_by query
+ # (which doesn't match all can_manage-able items, see #18865) --
+ # just ensure the current user actually has can_manage permission
+ # for the provided head_uuids, removing any that don't. At that
+ # point the caller's filters are an effective permission filter.
+ if @filters.include?(['link_class', '=', 'permission'])
+ @filters.map do |k|
+ if k[0] == 'tail_uuid' && k[1] == '=' && k[2] == current_user.uuid
+ @objects = Link.unscoped
+ elsif k[0] == 'head_uuid'
+ if k[1] == '=' && current_user.can?(manage: k[2])
+ @objects = Link.unscoped
+ elsif k[1] == 'in'
+ # Modify the filter operand element (k[2]) in place,
+ # removing any non-permitted UUIDs.
+ k[2].select! do |head_uuid|
+ current_user.can?(manage: head_uuid)
+ end
+ @objects = Link.unscoped
+ end
+ end
+ end
+ end
end
end
clnt
end
- def self.check_anonymous_user_token token
+ def self.check_anonymous_user_token(token:, remote:)
case token[0..2]
when 'v2/'
_, token_uuid, secret, optional = token.split('/')
secret = token
end
+ # Usually, the secret is salted
+ salted_secret = OpenSSL::HMAC.hexdigest('sha1', Rails.configuration.Users.AnonymousUserToken, remote)
+
+ # The anonymous token could be specified as a full v2 token in the config,
+ # but the config loader strips it down to the secret part.
# The anonymous token content and minimum length is verified in lib/config
- if secret.length >= 0 && secret == Rails.configuration.Users.AnonymousUserToken
+ if secret.length >= 0 && (secret == Rails.configuration.Users.AnonymousUserToken || secret == salted_secret)
return ApiClientAuthorization.new(user: User.find_by_uuid(anonymous_user_uuid),
uuid: Rails.configuration.ClusterID+"-gj3su-anonymouspublic",
- api_token: token,
+ api_token: secret,
api_client: anonymous_user_token_api_client,
scopes: ['GET /'])
else
return nil if token.nil? or token.empty?
remote ||= Rails.configuration.ClusterID
- auth = self.check_anonymous_user_token(token)
+ auth = self.check_anonymous_user_token(token: token, remote: remote)
if !auth.nil?
return auth
end
end
def self.default_orders
- ["#{table_name}.modified_at desc", "#{table_name}.uuid"]
+ ["#{table_name}.modified_at desc", "#{table_name}.uuid desc"]
end
def self.unique_columns
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+class FixCreatedAtIndexes < ActiveRecord::Migration[5.2]
+ @@idxtables = [:collections, :container_requests, :groups, :links, :repositories, :users, :virtual_machines, :workflows, :logs]
+
+ def up
+ @@idxtables.each do |table|
+ ActiveRecord::Base.connection.execute("DROP INDEX IF EXISTS index_#{table.to_s}_on_created_at")
+ ActiveRecord::Base.connection.execute("DROP INDEX IF EXISTS index_#{table.to_s}_on_created_at_uuid")
+ ActiveRecord::Base.connection.execute("DROP INDEX IF EXISTS index_#{table.to_s}_on_created_at_and_uuid")
+ ActiveRecord::Base.connection.execute("DROP INDEX IF EXISTS index_#{table.to_s}_on_modified_at")
+ ActiveRecord::Base.connection.execute("DROP INDEX IF EXISTS index_#{table.to_s}_on_modified_at_uuid")
+ ActiveRecord::Base.connection.execute("DROP INDEX IF EXISTS index_#{table.to_s}_on_modified_at_and_uuid")
+
+ ActiveRecord::Base.connection.execute("CREATE INDEX IF NOT EXISTS index_#{table.to_s}_on_created_at_and_uuid ON #{table.to_s} USING btree (created_at, uuid)")
+ ActiveRecord::Base.connection.execute("CREATE INDEX IF NOT EXISTS index_#{table.to_s}_on_modified_at_and_uuid ON #{table.to_s} USING btree (modified_at, uuid)")
+ end
+ end
+
+ def down
+ @@idxtables.each do |table|
+ ActiveRecord::Base.connection.execute("DROP INDEX IF EXISTS index_#{table.to_s}_on_modified_at_and_uuid")
+ ActiveRecord::Base.connection.execute("CREATE INDEX IF NOT EXISTS index_#{table.to_s}_on_modified_at_uuid ON #{table.to_s} USING btree (modified_at desc, uuid asc)")
+ end
+ end
+end
--
--- Name: index_collections_on_created_at; Type: INDEX; Schema: public; Owner: -
+-- Name: index_collections_on_created_at_and_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_collections_on_created_at ON public.collections USING btree (created_at);
+CREATE INDEX index_collections_on_created_at_and_uuid ON public.collections USING btree (created_at, uuid);
--
--
--- Name: index_collections_on_modified_at; Type: INDEX; Schema: public; Owner: -
+-- Name: index_collections_on_modified_at_and_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_collections_on_modified_at ON public.collections USING btree (modified_at);
-
-
---
--- Name: index_collections_on_modified_at_uuid; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX index_collections_on_modified_at_uuid ON public.collections USING btree (modified_at DESC, uuid);
+CREATE INDEX index_collections_on_modified_at_and_uuid ON public.collections USING btree (modified_at, uuid);
--
--
--- Name: index_container_requests_on_modified_at_uuid; Type: INDEX; Schema: public; Owner: -
+-- Name: index_container_requests_on_created_at_and_uuid; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX index_container_requests_on_created_at_and_uuid ON public.container_requests USING btree (created_at, uuid);
+
+
+--
+-- Name: index_container_requests_on_modified_at_and_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_container_requests_on_modified_at_uuid ON public.container_requests USING btree (modified_at DESC, uuid);
+CREATE INDEX index_container_requests_on_modified_at_and_uuid ON public.container_requests USING btree (modified_at, uuid);
--
--
--- Name: index_groups_on_created_at; Type: INDEX; Schema: public; Owner: -
+-- Name: index_groups_on_created_at_and_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_groups_on_created_at ON public.groups USING btree (created_at);
+CREATE INDEX index_groups_on_created_at_and_uuid ON public.groups USING btree (created_at, uuid);
--
--
--- Name: index_groups_on_modified_at; Type: INDEX; Schema: public; Owner: -
+-- Name: index_groups_on_modified_at_and_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_groups_on_modified_at ON public.groups USING btree (modified_at);
-
-
---
--- Name: index_groups_on_modified_at_uuid; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX index_groups_on_modified_at_uuid ON public.groups USING btree (modified_at DESC, uuid);
+CREATE INDEX index_groups_on_modified_at_and_uuid ON public.groups USING btree (modified_at, uuid);
--
--
--- Name: index_links_on_created_at; Type: INDEX; Schema: public; Owner: -
+-- Name: index_links_on_created_at_and_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_links_on_created_at ON public.links USING btree (created_at);
+CREATE INDEX index_links_on_created_at_and_uuid ON public.links USING btree (created_at, uuid);
--
--
--- Name: index_links_on_modified_at; Type: INDEX; Schema: public; Owner: -
+-- Name: index_links_on_modified_at_and_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_links_on_modified_at ON public.links USING btree (modified_at);
-
-
---
--- Name: index_links_on_modified_at_uuid; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX index_links_on_modified_at_uuid ON public.links USING btree (modified_at DESC, uuid);
+CREATE INDEX index_links_on_modified_at_and_uuid ON public.links USING btree (modified_at, uuid);
--
--
--- Name: index_logs_on_created_at; Type: INDEX; Schema: public; Owner: -
+-- Name: index_logs_on_created_at_and_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_logs_on_created_at ON public.logs USING btree (created_at);
+CREATE INDEX index_logs_on_created_at_and_uuid ON public.logs USING btree (created_at, uuid);
--
--
--- Name: index_logs_on_modified_at; Type: INDEX; Schema: public; Owner: -
+-- Name: index_logs_on_modified_at_and_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_logs_on_modified_at ON public.logs USING btree (modified_at);
-
-
---
--- Name: index_logs_on_modified_at_uuid; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX index_logs_on_modified_at_uuid ON public.logs USING btree (modified_at DESC, uuid);
+CREATE INDEX index_logs_on_modified_at_and_uuid ON public.logs USING btree (modified_at, uuid);
--
--
--- Name: index_repositories_on_modified_at_uuid; Type: INDEX; Schema: public; Owner: -
+-- Name: index_repositories_on_created_at_and_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_repositories_on_modified_at_uuid ON public.repositories USING btree (modified_at DESC, uuid);
+CREATE INDEX index_repositories_on_created_at_and_uuid ON public.repositories USING btree (created_at, uuid);
+
+
+--
+-- Name: index_repositories_on_modified_at_and_uuid; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX index_repositories_on_modified_at_and_uuid ON public.repositories USING btree (modified_at, uuid);
--
--
--- Name: index_users_on_created_at; Type: INDEX; Schema: public; Owner: -
+-- Name: index_users_on_created_at_and_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_users_on_created_at ON public.users USING btree (created_at);
+CREATE INDEX index_users_on_created_at_and_uuid ON public.users USING btree (created_at, uuid);
--
--
--- Name: index_users_on_modified_at; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX index_users_on_modified_at ON public.users USING btree (modified_at);
-
-
---
--- Name: index_users_on_modified_at_uuid; Type: INDEX; Schema: public; Owner: -
+-- Name: index_users_on_modified_at_and_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_users_on_modified_at_uuid ON public.users USING btree (modified_at DESC, uuid);
+CREATE INDEX index_users_on_modified_at_and_uuid ON public.users USING btree (modified_at, uuid);
--
CREATE UNIQUE INDEX index_users_on_uuid ON public.users USING btree (uuid);
+--
+-- Name: index_virtual_machines_on_created_at_and_uuid; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX index_virtual_machines_on_created_at_and_uuid ON public.virtual_machines USING btree (created_at, uuid);
+
+
--
-- Name: index_virtual_machines_on_hostname; Type: INDEX; Schema: public; Owner: -
--
--
--- Name: index_virtual_machines_on_modified_at_uuid; Type: INDEX; Schema: public; Owner: -
+-- Name: index_virtual_machines_on_modified_at_and_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_virtual_machines_on_modified_at_uuid ON public.virtual_machines USING btree (modified_at DESC, uuid);
+CREATE INDEX index_virtual_machines_on_modified_at_and_uuid ON public.virtual_machines USING btree (modified_at, uuid);
--
--
--- Name: index_workflows_on_modified_at_uuid; Type: INDEX; Schema: public; Owner: -
+-- Name: index_workflows_on_created_at_and_uuid; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX index_workflows_on_created_at_and_uuid ON public.workflows USING btree (created_at, uuid);
+
+
+--
+-- Name: index_workflows_on_modified_at_and_uuid; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX index_workflows_on_modified_at_uuid ON public.workflows USING btree (modified_at DESC, uuid);
+CREATE INDEX index_workflows_on_modified_at_and_uuid ON public.workflows USING btree (modified_at, uuid);
--
('20211027154300'),
('20220224203102'),
('20220301155729'),
-('20220303204419');
+('20220303204419'),
+('20220401153101');
cancelled_at: ~
cancelled_by_user_uuid: ~
cancelled_by_client_uuid: ~
- created_at: <%= 3.minute.ago.to_s(:db) %>
- started_at: <%= 3.minute.ago.to_s(:db) %>
+ created_at: <%= 2.7.minute.ago.to_s(:db) %>
+ started_at: <%= 2.7.minute.ago.to_s(:db) %>
finished_at: ~
script: hash
repository: active/foo
state: Ready
uuid: zzzzz-d1hrv-1yfj6xkidf2muk3
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
- created_at: <%= 3.1.minute.ago.to_s(:db) %>
+ created_at: <%= 2.9.minute.ago.to_s(:db) %>
components:
foo:
script: foo
# Helps test that clients cope with funny-shaped components.
# For an example, see #3321.
uuid: zzzzz-d1hrv-1yfj61234abcdk4
- created_at: <%= 2.minute.ago.to_s(:db) %>
+ created_at: <%= 4.minute.ago.to_s(:db) %>
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
state: Ready
uuid: zzzzz-d1hrv-1yfj61234abcdk3
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
- created_at: <%= 3.1.minute.ago.to_s(:db) %>
+ created_at: <%= 3.2.minute.ago.to_s(:db) %>
components:
part-one:
script_parameters:
uuid: zzzzz-d1hrv-1yfj6dcba4321k3
pipeline_template_uuid: zzzzz-p5p6p-aox0k0ofxrystgw
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
- created_at: <%= 3.1.minute.ago.to_s(:db) %>
+ created_at: <%= 3.3.minute.ago.to_s(:db) %>
components:
part-one:
script_parameters:
uuid: zzzzz-d1hrv-ri9dvgkgqs9y09j
owner_uuid: zzzzz-tpzed-0fusedrivertest
pipeline_template_uuid: zzzzz-p5p6p-vq4wuvy84xvaq2r
- created_at: 2014-09-15 12:00:00
+ created_at: 2014-09-16 12:00:00
name: "pipeline instance owned by FUSE"
components:
foo:
uuid: zzzzz-d1hrv-scarxiyajtshq3l
owner_uuid: zzzzz-j7d0g-0000ownedbyfuse
pipeline_template_uuid: zzzzz-p5p6p-vq4wuvy84xvaq2r
- created_at: 2014-09-15 12:00:00
+ created_at: 2014-09-17 12:00:00
name: "pipeline instance in FUSE project"
components:
foo:
state: Complete
uuid: zzzzz-d1hrv-ju5ghi0i9z2kqc6
owner_uuid: zzzzz-j7d0g-v955i6s2oi1cbso
- created_at: 2014-09-15 12:00:00
+ created_at: 2014-09-18 12:00:00
components:
foo:
script: foo
state: Complete
uuid: zzzzz-d1hrv-lihrbd0i9z2kqc6
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
- created_at: 2014-09-15 12:00:00
+ created_at: 2014-09-19 12:00:00
components:
foo:
script: foo
name: Pipeline in public project with other objects elsewhere
pipeline_template_uuid: zzzzz-p5p6p-aox0k0ofxrystgw
state: Complete
- created_at: 2014-09-15 12:00:00
+ created_at: 2014-09-20 12:00:00
components:
foo:
script: foo
name: Pipeline in New state in publicly accessible project
pipeline_template_uuid: zzzzz-p5p6p-tmpltpublicproj
state: New
- created_at: 2014-09-15 12:00:00
+ created_at: 2014-09-21 12:00:00
components:
foo:
script: foo
name: Pipeline in New state in public project with objects elsewhere
pipeline_template_uuid: zzzzz-p5p6p-aox0k0ofxrystgw
state: New
- created_at: 2014-09-15 12:00:00
+ created_at: 2014-09-22 12:00:00
components:
foo:
script: foo
name: Pipeline in public project in New state with file type data class with objects elsewhere
pipeline_template_uuid: zzzzz-p5p6p-aox0k0ofxrystgw
state: New
- created_at: 2014-09-15 12:00:00
+ created_at: 2014-09-23 12:00:00
components:
foo:
script: foo
name: running_with_job
uuid: zzzzz-d1hrv-runningpipeline
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
- created_at: <%= 3.1.minute.ago.to_s(:db) %>
- started_at: <%= 3.1.minute.ago.to_s(:db) %>
+ created_at: <%= 2.8.minute.ago.to_s(:db) %>
+ started_at: <%= 2.8.minute.ago.to_s(:db) %>
state: RunningOnServer
components:
foo:
uuid: zzzzz-d1hrv-twodonepipeline
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
state: Complete
- created_at: <%= 3.minute.ago.to_s(:db) %>
+ created_at: <%= 2.5.minute.ago.to_s(:db) %>
started_at: <%= 2.minute.ago.to_s(:db) %>
finished_at: <%= 1.minute.ago.to_s(:db) %>
components:
controller: 'logs',
}
assert_response :success
- assert_equal('logs.event_type asc, logs.modified_at desc, logs.uuid',
+ assert_equal('logs.event_type asc, logs.modified_at desc, logs.uuid desc',
assigns(:objects).order_values.join(', '))
end
controller: 'logs',
}
assert_response :success
- assert_equal('logs.modified_at asc, logs.uuid',
+ assert_equal('logs.modified_at asc, logs.uuid desc',
assigns(:objects).order_values.join(', '))
end
controller: 'logs',
}
assert_response :success
- assert_equal('logs.modified_at asc, logs.event_type desc, logs.uuid',
+ assert_equal('logs.modified_at asc, logs.event_type desc, logs.uuid desc',
assigns(:objects).order_values.join(', '))
end
# Should be able to read links directly too
get "/arvados/v1/links/#{can_read_uuid}",
- params: {},
headers: auth(:active)
assert_response :success
+ ### Create some objects of different types (other than projects)
+ ### inside a subproject inside the shared project, and share those
+ ### individual objects with a 3rd user ("spectator").
+ post '/arvados/v1/groups',
+ params: {
+ group: {
+ owner_uuid: groups(:public).uuid,
+ name: 'permission test subproject',
+ group_class: 'project',
+ },
+ },
+ headers: auth(:admin)
+ assert_response :success
+ subproject_uuid = json_response['uuid']
+
+ test_types = ['collection', 'workflow', 'container_request']
+ test_type_create_attrs = {
+ 'container_request' => {
+ command: ["echo", "foo"],
+ container_image: links(:docker_image_collection_tag).name,
+ cwd: "/tmp",
+ environment: {},
+ mounts: {"/out" => {kind: "tmp", capacity: 1000000}},
+ output_path: "/out",
+ runtime_constraints: {"vcpus" => 1, "ram" => 2},
+ },
+ }
+
+ test_object = {}
+ test_object_perm_link = {}
+ test_types.each do |test_type|
+ post "/arvados/v1/#{test_type}s",
+ params: {
+ test_type.to_sym => {
+ owner_uuid: subproject_uuid,
+ name: "permission test #{test_type} in subproject",
+ }.merge(test_type_create_attrs[test_type] || {}).to_json,
+ },
+ headers: auth(:admin)
+ assert_response :success
+ test_object[test_type] = json_response
+
+ post '/arvados/v1/links',
+ params: {
+ link: {
+ tail_uuid: users(:spectator).uuid,
+ link_class: 'permission',
+ name: 'can_read',
+ head_uuid: test_object[test_type]['uuid'],
+ }
+ },
+ headers: auth(:admin)
+ assert_response :success
+ test_object_perm_link[test_type] = json_response
+ end
+
+ # The "active-can_manage-project" permission should cause the
+ # "spectator-can_read-object" links to be visible to the "active"
+ # user.
+ test_types.each do |test_type|
+ get "/arvados/v1/permissions/#{test_object[test_type]['uuid']}",
+ headers: auth(:active)
+ assert_response :success
+ perm_uuids = json_response['items'].map { |item| item['uuid'] }
+ assert_includes perm_uuids, test_object_perm_link[test_type]['uuid'], "can_read_uuid not found"
+
+ get "/arvados/v1/links/#{test_object_perm_link[test_type]['uuid']}",
+ headers: auth(:active)
+ assert_response :success
+
+ [
+ ['head_uuid', '=', test_object[test_type]['uuid']],
+ ['head_uuid', 'in', [test_object[test_type]['uuid']]],
+ ['head_uuid', 'in', [users(:admin).uuid, test_object[test_type]['uuid']]],
+ ].each do |filter|
+ get "/arvados/v1/links",
+ params: {
+ filters: ([['link_class', '=', 'permission'], filter]).to_json,
+ },
+ headers: auth(:active)
+ assert_response :success
+ assert_not_empty json_response['items'], "could not find can_read link using index with filter #{filter}"
+ assert_equal test_object_perm_link[test_type]['uuid'], json_response['items'][0]['uuid']
+ end
+
+ # The "spectator-can_read-object" link should be visible to the
+ # subject user ("spectator") in a filter query, even without
+ # can_manage permission on the target object.
+ [
+ ['tail_uuid', '=', users(:spectator).uuid],
+ ].each do |filter|
+ get "/arvados/v1/links",
+ params: {
+ filters: ([['link_class', '=', 'permission'], filter]).to_json,
+ },
+ headers: auth(:spectator)
+ assert_response :success
+ perm_uuids = json_response['items'].map { |item| item['uuid'] }
+ assert_includes perm_uuids, test_object_perm_link[test_type]['uuid'], "could not find can_read link using index with filter #{filter}"
+ end
+ end
+
### Now delete the can_manage link
delete "/arvados/v1/links/#{can_manage_uuid}",
- params: nil,
headers: auth(:active)
assert_response :success
# Should not be able read these permission links again
- get "/arvados/v1/permissions/#{groups(:public).uuid}",
- params: nil,
+ test_types.each do |test_type|
+ get "/arvados/v1/permissions/#{groups(:public).uuid}",
+ headers: auth(:active)
+ assert_response 404
+
+ get "/arvados/v1/permissions/#{test_object[test_type]['uuid']}",
+ headers: auth(:active)
+ assert_response 404
+
+ get "/arvados/v1/links",
+ params: {
+ filters: [["link_class", "=", "permission"], ["head_uuid", "=", groups(:public).uuid]].to_json
+ },
+ headers: auth(:active)
+ assert_response :success
+ assert_equal [], json_response['items']
+
+ [
+ ['head_uuid', '=', test_object[test_type]['uuid']],
+ ['head_uuid', 'in', [users(:admin).uuid, test_object[test_type]['uuid']]],
+ ['head_uuid', 'in', []],
+ ].each do |filter|
+ get "/arvados/v1/links",
+ params: {
+ :filters => [["link_class", "=", "permission"], filter].to_json
+ },
+ headers: auth(:active)
+ assert_response :success
+ assert_equal [], json_response['items']
+ end
+
+ # Should not be able to read links directly either
+ get "/arvados/v1/links/#{can_read_uuid}",
+ headers: auth(:active)
+ assert_response 404
+
+ test_types.each do |test_type|
+ get "/arvados/v1/links/#{test_object_perm_link[test_type]['uuid']}",
+ headers: auth(:active)
+ assert_response 404
+ end
+ end
+
+ ### Create a collection, and share it with a direct permission
+ ### link (as opposed to sharing its parent project)
+ post "/arvados/v1/collections",
+ params: {
+ collection: {
+ name: 'permission test',
+ }
+ },
+ headers: auth(:admin)
+ assert_response :success
+ collection_uuid = json_response['uuid']
+ post "/arvados/v1/links",
+ params: {
+ link: {
+ tail_uuid: users(:spectator).uuid,
+ link_class: 'permission',
+ name: 'can_read',
+ head_uuid: collection_uuid,
+ properties: {}
+ }
+ },
+ headers: auth(:admin)
+ assert_response :success
+ can_read_collection_uuid = json_response['uuid']
+
+ # Should not be able read the permission link via permissions API,
+ # because permission is only can_read, not can_manage
+ get "/arvados/v1/permissions/#{collection_uuid}",
headers: auth(:active)
assert_response 404
- get "/arvados/v1/links",
- params: {
- :filters => [["link_class", "=", "permission"], ["head_uuid", "=", groups(:public).uuid]].to_json
- },
+ # Should not be able to read the permission link directly, for
+ # same reason
+ get "/arvados/v1/links/#{can_read_collection_uuid}",
headers: auth(:active)
+ assert_response 404
+
+ ### Now add a can_manage link
+ post "/arvados/v1/links",
+ params: {
+ link: {
+ tail_uuid: users(:active).uuid,
+ link_class: 'permission',
+ name: 'can_manage',
+ head_uuid: collection_uuid,
+ properties: {}
+ }
+ },
+ headers: auth(:admin)
assert_response :success
- assert_equal [], json_response['items']
+ can_manage_collection_uuid = json_response['uuid']
- # Should not be able to read links directly either
- get "/arvados/v1/links/#{can_read_uuid}",
- params: {},
+ # Should be able read both permission links via permissions API
+ get "/arvados/v1/permissions/#{collection_uuid}",
headers: auth(:active)
- assert_response 404
+ assert_response :success
+ perm_uuids = json_response['items'].map { |item| item['uuid'] }
+ assert_includes perm_uuids, can_read_collection_uuid, "can_read_uuid not found"
+ assert_includes perm_uuids, can_manage_collection_uuid, "can_manage_uuid not found"
+
+ # Should be able to read both permission links directly
+ [can_read_collection_uuid, can_manage_collection_uuid].each do |uuid|
+ get "/arvados/v1/links/#{uuid}",
+ headers: auth(:active)
+ assert_response :success
+ end
end
test "get_permissions returns 404 for nonexistent uuid" do
headers: auth(:admin)
assert_response :success
uuids = json_response['items'].collect { |i| i['uuid'] }
- assert_equal uuids, uuids.sort
+ assert_equal uuids, uuids.sort.reverse
end
def assert_link_classes_ascend(current_class, prev_class)
# Networking
# WEBUI PORT
- arv.vm.network "forwarded_port", guest: 9443, host: 9443
+ arv.vm.network "forwarded_port", guest: 443, host: 9443
# WORKBENCH1
- arv.vm.network "forwarded_port", guest: 9444, host: 9444
+ arv.vm.network "forwarded_port", guest: 8805, host: 9444
# WORKBENCH2
- arv.vm.network "forwarded_port", guest: 9445, host: 9445
+ arv.vm.network "forwarded_port", guest: 443, host: 9445
# KEEPPROXY
- arv.vm.network "forwarded_port", guest: 35101, host: 35101
+ arv.vm.network "forwarded_port", guest: 8801, host: 35101
# KEEPWEB
- arv.vm.network "forwarded_port", guest: 11002, host: 11002
+ arv.vm.network "forwarded_port", guest: 8802, host: 11002
# WEBSHELL
- arv.vm.network "forwarded_port", guest: 14202, host: 14202
+ arv.vm.network "forwarded_port", guest: 8803, host: 14202
# WEBSOCKET
- arv.vm.network "forwarded_port", guest: 18002, host: 18002
+ arv.vm.network "forwarded_port", guest: 8804, host: 18002
arv.vm.provision "shell",
inline: "cp -vr /vagrant/config_examples/single_host/single_hostname /home/vagrant/local_config_dir;
cp -vr /vagrant/tests /home/vagrant/tests;
# required to test with arvados-snakeoil certs
insecure: false
+ resources:
+ virtual_machines:
+ shell:
+ name: shell
+ backend: __SHELL_INT_IP__
+ port: 4200
+
### TOKENS
tokens:
system_root: __SYSTEM_ROOT_TOKEN__
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# This state tries to query the controller using the parameters set in
+# the `arvados.cluster.resources.virtual_machines` pillar, to get the
+# ARVADOS_VIRTUAL_MACHINE_UUID for the host and configure the arvados login-sync cron
+# as described in https://doc.arvados.org/main/install/install-shell-server.html
+
+{%- set curr_tpldir = tpldir %}
+{%- set tpldir = 'arvados' %}
+{%- from "arvados/map.jinja" import arvados with context %}
+{%- from "arvados/libtofs.jinja" import files_switch with context %}
+{%- set tpldir = curr_tpldir %}
+
+{%- set virtual_machines = arvados.cluster.resources.virtual_machines | default({}) %}
+{%- set api_token = arvados.cluster.tokens.system_root | yaml_encode %}
+{%- set api_host = arvados.cluster.Services.Controller.ExternalURL | regex_replace('^http(s?)://', '', ignorecase=true) %}
+
+extra_shell_cron_add_login_sync_add_jq_pkg_installed:
+ pkg.installed:
+ - name: jq
+
+{%- for vm, vm_params in virtual_machines.items() %}
+ {%- set vm_name = vm_params.name | default(vm) %}
+
+ # Check if any of the specified virtual_machines parameters corresponds to this instance
+ # It should be an error if we get more than one occurrence
+ {%- if vm_name in [grains['id'], grains['host'], grains['fqdn'], grains['nodename']] or
+ vm_params.backend in [grains['id'], grains['host'], grains['fqdn'], grains['nodename']] +
+ grains['ipv4'] + grains['ipv6'] %}
+
+ # We need to query the VM UUID
+ {%- set cmd_query_vm_uuid = 'arv --short virtual_machine list' ~
+ ' --filters \'[["hostname", "=", "' ~ vm_name ~ '"]]\''
+ %}
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_get_vm_uuid_cmd_run:
+ cmd.run:
+ - env:
+ - ARVADOS_API_TOKEN: {{ api_token }}
+ - ARVADOS_API_HOST: {{ api_host }}
+ - ARVADOS_API_HOST_INSECURE: {{ arvados.cluster.tls.insecure | default(false) }}
+ - name: {{ cmd_query_vm_uuid }} | head -1 | tee /tmp/vm_uuid_{{ vm }}
+ - unless:
+ - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
+ - require:
+ - gem: arvados-shell-package-install-gem-arvados-cli-installed
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_arvados_api_host_cron_env_present:
+ cron.env_present:
+ - name: ARVADOS_API_HOST
+ - value: {{ api_host }}
+ - onlyif:
+ - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_arvados_api_token_cron_env_present:
+ cron.env_present:
+ - name: ARVADOS_API_TOKEN
+ - value: {{ api_token }}
+ - onlyif:
+ - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_arvados_api_host_insecure_cron_env_present:
+ cron.env_present:
+ - name: ARVADOS_API_HOST_INSECURE
+ - value: {{ arvados.cluster.tls.insecure | default(false) }}
+ - onlyif:
+ - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_arvados_virtual_machine_uuid_cron_env_present:
+ cron.env_present:
+ - name: ARVADOS_VIRTUAL_MACHINE_UUID
+ - value: __slot__:salt:cmd.run("cat /tmp/vm_uuid_{{ vm }}")
+ - onlyif:
+ - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_arvados_login_sync_cron_present:
+ cron.present:
+ - name: /usr/local/bin/arvados-login-sync
+ - minute: '*/2'
+ - onlyif:
+ - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
+
+ {%- endif %}
+{%- endfor %}
## manage OS packages with some other tool and you don't want us messing up
## with your setup.
ruby:
-
## We set these to `true` here for testing purposes.
## They both default to `false`.
manage_ruby: true
virtual_machines:
shell:
name: webshell
- backend: 127.0.1.1
+ backend: 127.0.0.1
port: 4200
### TOKENS
resources:
virtual_machines:
shell:
- name: webshell
- backend: 127.0.1.1
+ name: __HOSTNAME_EXT__
+ backend: 127.0.0.1
port: 4200
### TOKENS
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# This state tries to query the controller using the parameters set in
+# the `arvados.cluster.resources.virtual_machines` pillar, to get the
+# ARVADOS_VIRTUAL_MACHINE_UUID for the host and configure the arvados login-sync cron
+# as described in https://doc.arvados.org/main/install/install-shell-server.html
+
+{%- set curr_tpldir = tpldir %}
+{%- set tpldir = 'arvados' %}
+{%- from "arvados/map.jinja" import arvados with context %}
+{%- from "arvados/libtofs.jinja" import files_switch with context %}
+{%- set tpldir = curr_tpldir %}
+
+{%- set virtual_machines = arvados.cluster.resources.virtual_machines | default({}) %}
+{%- set api_token = arvados.cluster.tokens.system_root | yaml_encode %}
+{%- set api_host = arvados.cluster.Services.Controller.ExternalURL | regex_replace('^http(s?)://', '', ignorecase=true) %}
+
+extra_shell_cron_add_login_sync_add_jq_pkg_installed:
+ pkg.installed:
+ - name: jq
+
+{%- for vm, vm_params in virtual_machines.items() %}
+ {%- set vm_name = vm_params.name | default(vm) %}
+
+ # Check if any of the specified virtual_machines parameters corresponds to this instance
+ # It should be an error if we get more than one occurrence
+ {%- if vm_name in [grains['id'], grains['host'], grains['fqdn'], grains['nodename']] or
+ vm_params.backend in [grains['id'], grains['host'], grains['fqdn'], grains['nodename']] +
+ grains['ipv4'] + grains['ipv6'] %}
+
+ # We need to query the VM UUID
+ {%- set cmd_query_vm_uuid = 'arv --short virtual_machine list' ~
+ ' --filters \'[["hostname", "=", "' ~ vm_name ~ '"]]\''
+ %}
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_get_vm_uuid_cmd_run:
+ cmd.run:
+ - env:
+ - ARVADOS_API_TOKEN: {{ api_token }}
+ - ARVADOS_API_HOST: {{ api_host }}
+ - ARVADOS_API_HOST_INSECURE: {{ arvados.cluster.tls.insecure | default(false) }}
+ - name: {{ cmd_query_vm_uuid }} | head -1 | tee /tmp/vm_uuid_{{ vm }}
+ - unless:
+ - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
+ - require:
+ - gem: arvados-shell-package-install-gem-arvados-cli-installed
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_arvados_api_host_cron_env_present:
+ cron.env_present:
+ - name: ARVADOS_API_HOST
+ - value: {{ api_host }}
+ - onlyif:
+ - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_arvados_api_token_cron_env_present:
+ cron.env_present:
+ - name: ARVADOS_API_TOKEN
+ - value: {{ api_token }}
+ - onlyif:
+ - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_arvados_api_host_insecure_cron_env_present:
+ cron.env_present:
+ - name: ARVADOS_API_HOST_INSECURE
+ - value: {{ arvados.cluster.tls.insecure | default(false) }}
+ - onlyif:
+ - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_arvados_virtual_machine_uuid_cron_env_present:
+ cron.env_present:
+ - name: ARVADOS_VIRTUAL_MACHINE_UUID
+ - value: __slot__:salt:cmd.run("cat /tmp/vm_uuid_{{ vm }}")
+ - onlyif:
+ - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
+
+extra_shell_cron_add_login_sync_add_{{ vm }}_arvados_login_sync_cron_present:
+ cron.present:
+ - name: /usr/local/bin/arvados-login-sync
+ - minute: '*/2'
+ - onlyif:
+ - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
+
+ {%- endif %}
+{%- endfor %}
# Same when using self-signed certificates.
SKIP_SNAKE_OIL="dont_add_snakeoil_certs"
fi
- for f in $(ls "${F_DIR}"/extra/extra/*.sls | egrep -v "${SKIP_SNAKE_OIL}|shell_sudo_passwordless"); do
+ for f in $(ls "${F_DIR}"/extra/extra/*.sls | egrep -v "${SKIP_SNAKE_OIL}|shell_"); do
echo " - extra.$(basename ${f} | sed 's/.sls$//g')" >> ${S_DIR}/top.sls
done
# Use byo or self-signed certificates
grep -q "custom_certs" ${S_DIR}/top.sls || echo " - extra.custom_certs" >> ${S_DIR}/top.sls
fi
- echo " - extra.shell_sudo_passwordless" >> ${S_DIR}/top.sls
echo " - postgres" >> ${S_DIR}/top.sls
echo " - docker.software" >> ${S_DIR}/top.sls
echo " - arvados" >> ${S_DIR}/top.sls
+ echo " - extra.shell_sudo_passwordless" >> ${S_DIR}/top.sls
+ echo " - extra.shell_cron_add_login_sync" >> ${S_DIR}/top.sls
# Pillars
echo " - docker" >> ${P_DIR}/top.sls
"shell")
# States
echo " - extra.shell_sudo_passwordless" >> ${S_DIR}/top.sls
+ echo " - extra.shell_cron_add_login_sync" >> ${S_DIR}/top.sls
grep -q "docker" ${S_DIR}/top.sls || echo " - docker.software" >> ${S_DIR}/top.sls
grep -q "arvados.${R}" ${S_DIR}/top.sls || echo " - arvados.${R}" >> ${S_DIR}/top.sls
# Pillars
projects / arvados.git / commitdiff