json (>= 1.8.0)
websocket-driver (0.6.5)
websocket-extensions (>= 0.1.0)
- websocket-extensions (0.1.3)
+ websocket-extensions (0.1.5)
xpath (2.1.0)
nokogiri (~> 1.3)
<meta name="author" content="">
<% if current_user %>
<% content_for :js do %>
- window.defaultSession = <%=raw({baseURL: Rails.configuration.Services.Controller.ExternalURL.to_s.gsub(/\/?$/,'/'), token: Thread.current[:arvados_api_token], user: current_user}.to_json)%>
+ window.defaultSession = <%=raw({baseURL: Rails.configuration.Services.Controller.ExternalURL.to_s.sub(/\/*$/,'/'), token: Thread.current[:arvados_api_token], user: current_user}.to_json)%>
<% end %>
<% end %>
<% if current_user and $arvados_api_client.discovery[:websocketUrl] %>
<%= yield :head %>
<%= javascript_tag do %>
angular.module('Arvados').value('arvadosApiToken', '<%=Thread.current[:arvados_api_token]%>');
- angular.module('Arvados').value('arvadosDiscoveryUri', '<%= Rails.configuration.Services.Controller.ExternalURL.to_s + '/discovery/v1/apis/arvados/v1/rest' %>');
+ angular.module('Arvados').value('arvadosDiscoveryUri', '<%= Rails.configuration.Services.Controller.ExternalURL.to_s.sub(/\/*$/,'/') + 'discovery/v1/apis/arvados/v1/rest' %>');
<%= yield :js %>
<% end %>
<style>
. build/run-library.sh
-python_sdk_ts=$(cd sdk/python && timestamp_from_git)
-cwl_runner_ts=$(cd sdk/cwl && timestamp_from_git)
-
-python_sdk_version=$(cd sdk/python && nohash_version_from_git 0.1)
-cwl_runner_version=$(cd sdk/cwl && nohash_version_from_git 1.0)
-
-if [[ $python_sdk_ts -gt $cwl_runner_ts ]]; then
- cwl_runner_version=$(cd sdk/python && nohash_version_from_git 1.0)
-fi
+calculate_python_sdk_cwl_package_versions
set -x
docker build --no-cache --build-arg sdk=$sdk --build-arg runner=$runner --build-arg salad=$salad --build-arg cwltool=$cwltool --build-arg pythoncmd=$py --build-arg pipcmd=$pipcmd -f "$WORKSPACE/sdk/dev-jobs.dockerfile" -t arvados/jobs:$cwl_runner_version "$WORKSPACE/sdk"
ARVADOS_BUILDING_ITERATION="1"
fi
-python_sdk_ts=$(cd sdk/python && timestamp_from_git)
-cwl_runner_ts=$(cd sdk/cwl && timestamp_from_git)
-
-python_sdk_version=$(cd sdk/python && nohash_version_from_git 0.1)
-cwl_runner_version=$(cd sdk/cwl && nohash_version_from_git 1.0)
-
-if [[ $python_sdk_ts -gt $cwl_runner_ts ]]; then
- cwl_runner_version=$(cd sdk/python && nohash_version_from_git 1.0)
-fi
+calculate_python_sdk_cwl_package_versions
echo cwl_runner_version $cwl_runner_version python_sdk_version $python_sdk_version
# Output the version being built, or if we're building a
# dev/prerelease, output a version number based on the git log for
# the given $subdir.
- local minorversion="$1"; shift # unused
local subdir="$1"; shift
if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
echo "$ARVADOS_BUILDING_VERSION"
echo "$ARVADOS_BUILDING_VERSION"
return
fi
- version_from_git $1 | cut -d. -f1-4
+ version_from_git | cut -d. -f1-4
}
timestamp_from_git() {
format_last_commit_here "%ct" "$subdir"
}
+calculate_python_sdk_cwl_package_versions() {
+ python_sdk_ts=$(cd sdk/python && timestamp_from_git)
+ cwl_runner_ts=$(cd sdk/cwl && timestamp_from_git)
+
+ python_sdk_version=$(cd sdk/python && nohash_version_from_git)
+ cwl_runner_version=$(cd sdk/cwl && nohash_version_from_git)
+
+ if [[ $python_sdk_ts -gt $cwl_runner_ts ]]; then
+ cwl_runner_version=$python_sdk_version
+ fi
+}
+
handle_python_package () {
# This function assumes the current working directory is the python package directory
if [ -n "$(find dist -name "*-$(nohash_version_from_git).tar.gz" -print -quit)" ]; then
cd "$WORKSPACE"
ts="$(timestamp_from_git "$dir")"
if [[ "$ts" -gt "$timestamp" ]]; then
- version=$(version_from_git "" "$dir")
+ version=$(version_from_git "$dir")
timestamp="$ts"
fi
done
"git.arvados.org/arvados.git/lib/crunchrun"
"git.arvados.org/arvados.git/lib/dispatchcloud"
"git.arvados.org/arvados.git/lib/install"
+ "git.arvados.org/arvados.git/lib/undelete"
"git.arvados.org/arvados.git/services/ws"
)
"crunch-run": crunchrun.Command,
"dispatch-cloud": dispatchcloud.Command,
"install": install.Command,
+ "undelete": undelete.Command,
"ws": ws.Command,
})
)
|keep-block-check|Given a list of keep block locators, check that each block exists on one of the configured keepstore servers and verify the block hash.|
|keep-exercise|Benchmarking tool to test throughput and reliability of keepstores under various usage patterns.|
|keep-rsync|Get lists of blocks from two clusters, copy blocks which exist on source cluster but are missing from destination cluster.|
-|sync-groups|Take a CSV file listing (group, username) pairs and synchronize membership in Arvados groups.|
+|sync-groups|Take a CSV file listing with (group, user, permission) records and synchronize membership in Arvados groups.|
Select one of the following login mechanisms for your cluster.
# If all users will authenticate with Google, "configure Google login":#google.
+# If all users will authenticate with an OpenID Connect provider (other than Google), "configure OpenID Connect":#oidc.
# If all users will authenticate with an existing LDAP service, "configure LDAP":#ldap.
# If all users will authenticate using PAM as configured on your controller node, "configure PAM":#pam.
ClientSecret: "zzzzzzzzzzzzzzzzzzzzzzzz"
</pre>
+h2(#oidc). OpenID Connect
+
+With this configuration, users will sign in with a third-party OpenID Connect provider. The provider will supply appropriate values for the issuer URL, client ID, and client secret config entries.
+
+<pre>
+ Login:
+ OpenIDConnect:
+ Enable: true
+ Issuer: https://accounts.example.com/
+ ClientID: "0123456789abcdef"
+ ClientSecret: "zzzzzzzzzzzzzzzzzzzzzzzz"
+</pre>
+
h2(#ldap). LDAP
With this configuration, authentication uses an external LDAP service like OpenLDAP or Active Directory.
This tool reads a CSV (comma-separated values) file having information about external groups and their members. When running it for the first time, it'll create a special group named 'Externally synchronized groups' meant to be the parent of all the remote groups.
-Every line on the file should have 2 values: a group name and a local user identifier, meaning that the named user is a member of the group. The tool will create the group if it doesn't exist, and add the user to it. If group member is not present on the input file, the account will be removed from the group.
+Every line on the file should have 3 values: a group name, a local user identifier and a permission level, meaning that the named user is a member of the group with the provided permission. The tool will create the group if it doesn't exist, and add the user to it. If any group member is not present on the input file, it will be removed from the group.
Users can be identified by their email address or username: the tool will check if every user exist on the system, and report back when not found. Groups on the other hand, are identified by their name.
+Permission level can be one of the following: @can_read@, @can_write@ or @can_manage@, giving the group member read, read/write or managing privileges on the group. For backwards compatibility purposes, if any record omits the third (permission) field, it will default to @can_write@ permission. You can read more about permissions on the "group management admin guide":/admin/group-management.html.
+
This tool is designed to be run periodically reading a file created by a remote auth system (ie: LDAP) dump script, applying what's included on the file as the source of truth.
# work. If false, only the primary email address will be used.
AlternateEmailAddresses: true
+ OpenIDConnect:
+ # Authenticate with an OpenID Connect provider.
+ Enable: false
+
+ # Issuer URL, e.g., "https://login.example.com".
+ #
+ # This must be exactly equal to the URL returned by the issuer
+ # itself in its config response ("isser" key). If the
+ # configured value is "https://example" and the provider
+ # returns "https://example:443" or "https://example/" then
+ # login will fail, even though those URLs are equivalent
+ # (RFC3986).
+ Issuer: ""
+
+ # Your client ID and client secret (supplied by the provider).
+ ClientID: ""
+ ClientSecret: ""
+
PAM:
# (Experimental) Use PAM to authenticate users.
Enable: false
# "ou=Users,dc=example,dc=com"
SearchBase: ""
- # Additional filters for username lookup. Special characters
- # in assertion values must be escaped (see RFC4515). Example:
- # "(objectClass=person)"
+ # Additional filters to apply when looking up users' LDAP
+ # entries. This can be used to restrict access to a subset of
+ # LDAP users, or to disambiguate users from other directory
+ # entries that have the SearchAttribute present.
+ #
+ # Special characters in assertion values must be escaped (see
+ # RFC4515).
+ #
+ # Example: "(objectClass=person)"
SearchFilters: ""
# LDAP attribute to use as the user's email address.
"Login.LDAP.URL": false,
"Login.LDAP.UsernameAttribute": false,
"Login.LoginCluster": true,
+ "Login.OpenIDConnect": true,
+ "Login.OpenIDConnect.ClientID": false,
+ "Login.OpenIDConnect.ClientSecret": false,
+ "Login.OpenIDConnect.Enable": true,
+ "Login.OpenIDConnect.Issuer": false,
"Login.PAM": true,
"Login.PAM.DefaultEmailDomain": false,
"Login.PAM.Enable": true,
# work. If false, only the primary email address will be used.
AlternateEmailAddresses: true
+ OpenIDConnect:
+ # Authenticate with an OpenID Connect provider.
+ Enable: false
+
+ # Issuer URL, e.g., "https://login.example.com".
+ #
+ # This must be exactly equal to the URL returned by the issuer
+ # itself in its config response ("isser" key). If the
+ # configured value is "https://example" and the provider
+ # returns "https://example:443" or "https://example/" then
+ # login will fail, even though those URLs are equivalent
+ # (RFC3986).
+ Issuer: ""
+
+ # Your client ID and client secret (supplied by the provider).
+ ClientID: ""
+ ClientSecret: ""
+
PAM:
# (Experimental) Use PAM to authenticate users.
Enable: false
# "ou=Users,dc=example,dc=com"
SearchBase: ""
- # Additional filters for username lookup. Special characters
- # in assertion values must be escaped (see RFC4515). Example:
- # "(objectClass=person)"
+ # Additional filters to apply when looking up users' LDAP
+ # entries. This can be used to restrict access to a subset of
+ # LDAP users, or to disambiguate users from other directory
+ # entries that have the SearchAttribute present.
+ #
+ # Special characters in assertion values must be escaped (see
+ # RFC4515).
+ #
+ # Example: "(objectClass=person)"
SearchFilters: ""
# LDAP attribute to use as the user's email address.
// // ldr.Path == "/tmp/c.yaml"
func (ldr *Loader) SetupFlags(flagset *flag.FlagSet) {
flagset.StringVar(&ldr.Path, "config", arvados.DefaultConfigFile, "Site configuration `file` (default may be overridden by setting an ARVADOS_CONFIG environment variable)")
- flagset.StringVar(&ldr.KeepstorePath, "legacy-keepstore-config", defaultKeepstoreConfigPath, "Legacy keepstore configuration `file`")
- flagset.StringVar(&ldr.KeepWebPath, "legacy-keepweb-config", defaultKeepWebConfigPath, "Legacy keep-web configuration `file`")
- flagset.StringVar(&ldr.CrunchDispatchSlurmPath, "legacy-crunch-dispatch-slurm-config", defaultCrunchDispatchSlurmConfigPath, "Legacy crunch-dispatch-slurm configuration `file`")
- flagset.StringVar(&ldr.WebsocketPath, "legacy-ws-config", defaultWebsocketConfigPath, "Legacy arvados-ws configuration `file`")
- flagset.StringVar(&ldr.KeepproxyPath, "legacy-keepproxy-config", defaultKeepproxyConfigPath, "Legacy keepproxy configuration `file`")
- flagset.StringVar(&ldr.GitHttpdPath, "legacy-git-httpd-config", defaultGitHttpdConfigPath, "Legacy arv-git-httpd configuration `file`")
- flagset.StringVar(&ldr.KeepBalancePath, "legacy-keepbalance-config", defaultKeepBalanceConfigPath, "Legacy keep-balance configuration `file`")
- flagset.BoolVar(&ldr.SkipLegacy, "skip-legacy", false, "Don't load legacy config files")
+ if !ldr.SkipLegacy {
+ flagset.StringVar(&ldr.KeepstorePath, "legacy-keepstore-config", defaultKeepstoreConfigPath, "Legacy keepstore configuration `file`")
+ flagset.StringVar(&ldr.KeepWebPath, "legacy-keepweb-config", defaultKeepWebConfigPath, "Legacy keep-web configuration `file`")
+ flagset.StringVar(&ldr.CrunchDispatchSlurmPath, "legacy-crunch-dispatch-slurm-config", defaultCrunchDispatchSlurmConfigPath, "Legacy crunch-dispatch-slurm configuration `file`")
+ flagset.StringVar(&ldr.WebsocketPath, "legacy-ws-config", defaultWebsocketConfigPath, "Legacy arvados-ws configuration `file`")
+ flagset.StringVar(&ldr.KeepproxyPath, "legacy-keepproxy-config", defaultKeepproxyConfigPath, "Legacy keepproxy configuration `file`")
+ flagset.StringVar(&ldr.GitHttpdPath, "legacy-git-httpd-config", defaultGitHttpdConfigPath, "Legacy arv-git-httpd configuration `file`")
+ flagset.StringVar(&ldr.KeepBalancePath, "legacy-keepbalance-config", defaultKeepBalanceConfigPath, "Legacy keep-balance configuration `file`")
+ flagset.BoolVar(&ldr.SkipLegacy, "skip-legacy", false, "Don't load legacy config files")
+ }
}
// MungeLegacyConfigArgs checks args for a -config flag whose argument
req := httptest.NewRequest(method, "/arvados/v1/config", nil)
resp := httptest.NewRecorder()
s.handler.ServeHTTP(resp, req)
- c.Check(resp.Code, check.Equals, http.StatusOK)
+ c.Log(resp.Body.String())
+ if !c.Check(resp.Code, check.Equals, http.StatusOK) {
+ continue
+ }
c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, `*`)
c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Matches, `.*\bGET\b.*`)
c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Matches, `.+`)
continue
}
var cluster arvados.Cluster
- c.Log(resp.Body.String())
err := json.Unmarshal(resp.Body.Bytes(), &cluster)
c.Check(err, check.IsNil)
c.Check(cluster.ManagementToken, check.Equals, "")
c.Check(cluster.SystemRootToken, check.Equals, "")
- c.Check(cluster.Collections.BlobSigning, check.DeepEquals, true)
+ c.Check(cluster.Collections.BlobSigning, check.Equals, true)
c.Check(cluster.Collections.BlobSigningTTL, check.Equals, arvados.Duration(23*time.Second))
}
}
func chooseLoginController(cluster *arvados.Cluster, railsProxy *railsProxy) loginController {
wantGoogle := cluster.Login.Google.Enable
+ wantOpenIDConnect := cluster.Login.OpenIDConnect.Enable
wantSSO := cluster.Login.SSO.Enable
wantPAM := cluster.Login.PAM.Enable
wantLDAP := cluster.Login.LDAP.Enable
switch {
- case wantGoogle && !wantSSO && !wantPAM && !wantLDAP:
- return &googleLoginController{Cluster: cluster, RailsProxy: railsProxy}
- case !wantGoogle && wantSSO && !wantPAM && !wantLDAP:
+ case wantGoogle && !wantOpenIDConnect && !wantSSO && !wantPAM && !wantLDAP:
+ return &oidcLoginController{
+ Cluster: cluster,
+ RailsProxy: railsProxy,
+ Issuer: "https://accounts.google.com",
+ ClientID: cluster.Login.Google.ClientID,
+ ClientSecret: cluster.Login.Google.ClientSecret,
+ UseGooglePeopleAPI: cluster.Login.Google.AlternateEmailAddresses,
+ }
+ case !wantGoogle && wantOpenIDConnect && !wantSSO && !wantPAM && !wantLDAP:
+ return &oidcLoginController{
+ Cluster: cluster,
+ RailsProxy: railsProxy,
+ Issuer: cluster.Login.OpenIDConnect.Issuer,
+ ClientID: cluster.Login.OpenIDConnect.ClientID,
+ ClientSecret: cluster.Login.OpenIDConnect.ClientSecret,
+ }
+ case !wantGoogle && !wantOpenIDConnect && wantSSO && !wantPAM && !wantLDAP:
return &ssoLoginController{railsProxy}
- case !wantGoogle && !wantSSO && wantPAM && !wantLDAP:
+ case !wantGoogle && !wantOpenIDConnect && !wantSSO && wantPAM && !wantLDAP:
return &pamLoginController{Cluster: cluster, RailsProxy: railsProxy}
- case !wantGoogle && !wantSSO && !wantPAM && wantLDAP:
+ case !wantGoogle && !wantOpenIDConnect && !wantSSO && !wantPAM && wantLDAP:
return &ldapLoginController{Cluster: cluster, RailsProxy: railsProxy}
default:
return errorLoginController{
- error: errors.New("configuration problem: exactly one of Login.Google, Login.SSO, Login.PAM, and Login.LDAP must be enabled"),
+ error: errors.New("configuration problem: exactly one of Login.Google, Login.OpenIDConnect, Login.SSO, Login.PAM, and Login.LDAP must be enabled"),
}
}
}
"google.golang.org/api/people/v1"
)
-type googleLoginController struct {
- Cluster *arvados.Cluster
- RailsProxy *railsProxy
+type oidcLoginController struct {
+ Cluster *arvados.Cluster
+ RailsProxy *railsProxy
+ Issuer string // OIDC issuer URL, e.g., "https://accounts.google.com"
+ ClientID string
+ ClientSecret string
+ UseGooglePeopleAPI bool // Use Google People API to look up alternate email addresses
- issuer string // override OIDC issuer URL (normally https://accounts.google.com) for testing
- peopleAPIBasePath string // override Google People API base URL (normally set by google pkg to https://people.googleapis.com/)
- provider *oidc.Provider
- mu sync.Mutex
+ // override Google People API base URL for testing purposes
+ // (normally empty, set by google pkg to
+ // https://people.googleapis.com/)
+ peopleAPIBasePath string
+
+ provider *oidc.Provider // initialized by setup()
+ oauth2conf *oauth2.Config // initialized by setup()
+ verifier *oidc.IDTokenVerifier // initialized by setup()
+ mu sync.Mutex // protects setup()
}
-func (ctrl *googleLoginController) getProvider() (*oidc.Provider, error) {
+// Initialize ctrl.provider and ctrl.oauth2conf.
+func (ctrl *oidcLoginController) setup() error {
ctrl.mu.Lock()
defer ctrl.mu.Unlock()
- if ctrl.provider == nil {
- issuer := ctrl.issuer
- if issuer == "" {
- issuer = "https://accounts.google.com"
- }
- provider, err := oidc.NewProvider(context.Background(), issuer)
- if err != nil {
- return nil, err
- }
- ctrl.provider = provider
+ if ctrl.provider != nil {
+ // already set up
+ return nil
}
- return ctrl.provider, nil
-}
-
-func (ctrl *googleLoginController) Logout(ctx context.Context, opts arvados.LogoutOptions) (arvados.LogoutResponse, error) {
- return noopLogout(ctrl.Cluster, opts)
-}
-
-func (ctrl *googleLoginController) Login(ctx context.Context, opts arvados.LoginOptions) (arvados.LoginResponse, error) {
- provider, err := ctrl.getProvider()
+ redirURL, err := (*url.URL)(&ctrl.Cluster.Services.Controller.ExternalURL).Parse("/" + arvados.EndpointLogin.Path)
if err != nil {
- return loginError(fmt.Errorf("error setting up OpenID Connect provider: %s", err))
+ return fmt.Errorf("error making redirect URL: %s", err)
}
- redirURL, err := (*url.URL)(&ctrl.Cluster.Services.Controller.ExternalURL).Parse("/login")
+ provider, err := oidc.NewProvider(context.Background(), ctrl.Issuer)
if err != nil {
- return loginError(fmt.Errorf("error making redirect URL: %s", err))
+ return err
}
- conf := &oauth2.Config{
- ClientID: ctrl.Cluster.Login.Google.ClientID,
- ClientSecret: ctrl.Cluster.Login.Google.ClientSecret,
+ ctrl.oauth2conf = &oauth2.Config{
+ ClientID: ctrl.ClientID,
+ ClientSecret: ctrl.ClientSecret,
Endpoint: provider.Endpoint(),
Scopes: []string{oidc.ScopeOpenID, "profile", "email"},
RedirectURL: redirURL.String(),
}
- verifier := provider.Verifier(&oidc.Config{
- ClientID: conf.ClientID,
+ ctrl.verifier = provider.Verifier(&oidc.Config{
+ ClientID: ctrl.ClientID,
})
+ ctrl.provider = provider
+ return nil
+}
+
+func (ctrl *oidcLoginController) Logout(ctx context.Context, opts arvados.LogoutOptions) (arvados.LogoutResponse, error) {
+ return noopLogout(ctrl.Cluster, opts)
+}
+
+func (ctrl *oidcLoginController) Login(ctx context.Context, opts arvados.LoginOptions) (arvados.LoginResponse, error) {
+ err := ctrl.setup()
+ if err != nil {
+ return loginError(fmt.Errorf("error setting up OpenID Connect provider: %s", err))
+ }
if opts.State == "" {
- // Initiate Google sign-in.
+ // Initiate OIDC sign-in.
if opts.ReturnTo == "" {
return loginError(errors.New("missing return_to parameter"))
}
- me := url.URL(ctrl.Cluster.Services.Controller.ExternalURL)
- callback, err := me.Parse("/" + arvados.EndpointLogin.Path)
- if err != nil {
- return loginError(err)
- }
- conf.RedirectURL = callback.String()
state := ctrl.newOAuth2State([]byte(ctrl.Cluster.SystemRootToken), opts.Remote, opts.ReturnTo)
return arvados.LoginResponse{
- RedirectLocation: conf.AuthCodeURL(state.String(),
+ RedirectLocation: ctrl.oauth2conf.AuthCodeURL(state.String(),
// prompt=select_account tells Google
// to show the "choose which Google
// account" page, even if the client
oauth2.SetAuthURLParam("prompt", "select_account")),
}, nil
} else {
- // Callback after Google sign-in.
+ // Callback after OIDC sign-in.
state := ctrl.parseOAuth2State(opts.State)
if !state.verify([]byte(ctrl.Cluster.SystemRootToken)) {
return loginError(errors.New("invalid OAuth2 state"))
}
- oauth2Token, err := conf.Exchange(ctx, opts.Code)
+ oauth2Token, err := ctrl.oauth2conf.Exchange(ctx, opts.Code)
if err != nil {
return loginError(fmt.Errorf("error in OAuth2 exchange: %s", err))
}
if !ok {
return loginError(errors.New("error in OAuth2 exchange: no ID token in OAuth2 token"))
}
- idToken, err := verifier.Verify(ctx, rawIDToken)
+ idToken, err := ctrl.verifier.Verify(ctx, rawIDToken)
if err != nil {
return loginError(fmt.Errorf("error verifying ID token: %s", err))
}
- authinfo, err := ctrl.getAuthInfo(ctx, ctrl.Cluster, conf, oauth2Token, idToken)
+ authinfo, err := ctrl.getAuthInfo(ctx, oauth2Token, idToken)
if err != nil {
return loginError(err)
}
}
}
-func (ctrl *googleLoginController) UserAuthenticate(ctx context.Context, opts arvados.UserAuthenticateOptions) (arvados.APIClientAuthorization, error) {
+func (ctrl *oidcLoginController) UserAuthenticate(ctx context.Context, opts arvados.UserAuthenticateOptions) (arvados.APIClientAuthorization, error) {
return arvados.APIClientAuthorization{}, httpserver.ErrorWithStatus(errors.New("username/password authentication is not available"), http.StatusBadRequest)
}
// primary address at index 0. The provided defaultAddr is always
// included in the returned slice, and is used as the primary if the
// Google API does not indicate one.
-func (ctrl *googleLoginController) getAuthInfo(ctx context.Context, cluster *arvados.Cluster, conf *oauth2.Config, token *oauth2.Token, idToken *oidc.IDToken) (*rpc.UserSessionAuthInfo, error) {
+func (ctrl *oidcLoginController) getAuthInfo(ctx context.Context, token *oauth2.Token, idToken *oidc.IDToken) (*rpc.UserSessionAuthInfo, error) {
var ret rpc.UserSessionAuthInfo
defer ctxlog.FromContext(ctx).WithField("ret", &ret).Debug("getAuthInfo returned")
ret.Email = claims.Email
}
- if !ctrl.Cluster.Login.Google.AlternateEmailAddresses {
+ if !ctrl.UseGooglePeopleAPI {
if ret.Email == "" {
return nil, fmt.Errorf("cannot log in with unverified email address %q", claims.Email)
}
return &ret, nil
}
- svc, err := people.NewService(ctx, option.WithTokenSource(conf.TokenSource(ctx, token)), option.WithScopes(people.UserEmailsReadScope))
+ svc, err := people.NewService(ctx, option.WithTokenSource(ctrl.oauth2conf.TokenSource(ctx, token)), option.WithScopes(people.UserEmailsReadScope))
if err != nil {
return nil, fmt.Errorf("error setting up People API: %s", err)
}
return
}
-func (ctrl *googleLoginController) newOAuth2State(key []byte, remote, returnTo string) oauth2State {
+func (ctrl *oidcLoginController) newOAuth2State(key []byte, remote, returnTo string) oauth2State {
s := oauth2State{
Time: time.Now().Unix(),
Remote: remote,
ReturnTo string // redirect target
}
-func (ctrl *googleLoginController) parseOAuth2State(encoded string) (s oauth2State) {
+func (ctrl *oidcLoginController) parseOAuth2State(encoded string) (s oauth2State) {
// Errors are not checked. If decoding/parsing fails, the
// token will be rejected by verify().
decoded, _ := base64.RawURLEncoding.DecodeString(encoded)
"context"
"crypto/rand"
"crypto/rsa"
+ "encoding/base64"
"encoding/json"
"fmt"
"net/http"
check.TestingT(t)
}
-var _ = check.Suite(&LoginSuite{})
+var _ = check.Suite(&OIDCLoginSuite{})
-type LoginSuite struct {
+type OIDCLoginSuite struct {
cluster *arvados.Cluster
ctx context.Context
localdb *Conn
issuerKey *rsa.PrivateKey
// expected token request
- validCode string
+ validCode string
+ validClientID string
+ validClientSecret string
// desired response from token endpoint
authEmail string
authEmailVerified bool
authName string
}
-func (s *LoginSuite) TearDownSuite(c *check.C) {
+func (s *OIDCLoginSuite) TearDownSuite(c *check.C) {
// Undo any changes/additions to the user database so they
// don't affect subsequent tests.
arvadostest.ResetEnv()
c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil)
}
-func (s *LoginSuite) SetUpTest(c *check.C) {
+func (s *OIDCLoginSuite) SetUpTest(c *check.C) {
var err error
s.issuerKey, err = rsa.GenerateKey(rand.Reader, 2048)
c.Assert(err, check.IsNil)
"userinfo_endpoint": s.fakeIssuer.URL + "/userinfo",
})
case "/token":
+ var clientID, clientSecret string
+ auth, _ := base64.StdEncoding.DecodeString(strings.TrimPrefix(req.Header.Get("Authorization"), "Basic "))
+ authsplit := strings.Split(string(auth), ":")
+ if len(authsplit) == 2 {
+ clientID, _ = url.QueryUnescape(authsplit[0])
+ clientSecret, _ = url.QueryUnescape(authsplit[1])
+ }
+ if clientID != s.validClientID || clientSecret != s.validClientSecret {
+ c.Logf("fakeIssuer: expected (%q, %q) got (%q, %q)", s.validClientID, s.validClientSecret, clientID, clientSecret)
+ w.WriteHeader(http.StatusUnauthorized)
+ return
+ }
+
if req.Form.Get("code") != s.validCode || s.validCode == "" {
w.WriteHeader(http.StatusUnauthorized)
return
}
idToken, _ := json.Marshal(map[string]interface{}{
"iss": s.fakeIssuer.URL,
- "aud": []string{"test%client$id"},
+ "aud": []string{clientID},
"sub": "fake-user-id",
- "exp": time.Now().UTC().Add(time.Minute).UnixNano(),
- "iat": time.Now().UTC().UnixNano(),
+ "exp": time.Now().UTC().Add(time.Minute).Unix(),
+ "iat": time.Now().UTC().Unix(),
"nonce": "fake-nonce",
"email": s.authEmail,
"email_verified": s.authEmailVerified,
s.fakePeopleAPIResponse = map[string]interface{}{}
cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
+ c.Assert(err, check.IsNil)
s.cluster, err = cfg.GetCluster("")
+ c.Assert(err, check.IsNil)
s.cluster.Login.SSO.Enable = false
s.cluster.Login.Google.Enable = true
s.cluster.Login.Google.ClientID = "test%client$id"
s.cluster.Login.Google.ClientSecret = "test#client/secret"
s.cluster.Users.PreferDomainForUsername = "PreferDomainForUsername.example.com"
- c.Assert(err, check.IsNil)
+ s.validClientID = "test%client$id"
+ s.validClientSecret = "test#client/secret"
s.localdb = NewConn(s.cluster)
- s.localdb.loginController.(*googleLoginController).issuer = s.fakeIssuer.URL
- s.localdb.loginController.(*googleLoginController).peopleAPIBasePath = s.fakePeopleAPI.URL
+ c.Assert(s.localdb.loginController, check.FitsTypeOf, (*oidcLoginController)(nil))
+ s.localdb.loginController.(*oidcLoginController).Issuer = s.fakeIssuer.URL
+ s.localdb.loginController.(*oidcLoginController).peopleAPIBasePath = s.fakePeopleAPI.URL
s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
*s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)
}
-func (s *LoginSuite) TearDownTest(c *check.C) {
+func (s *OIDCLoginSuite) TearDownTest(c *check.C) {
s.railsSpy.Close()
}
-func (s *LoginSuite) TestGoogleLogout(c *check.C) {
+func (s *OIDCLoginSuite) TestGoogleLogout(c *check.C) {
resp, err := s.localdb.Logout(context.Background(), arvados.LogoutOptions{ReturnTo: "https://foo.example.com/bar"})
c.Check(err, check.IsNil)
c.Check(resp.RedirectLocation, check.Equals, "https://foo.example.com/bar")
}
-func (s *LoginSuite) TestGoogleLogin_Start_Bogus(c *check.C) {
+func (s *OIDCLoginSuite) TestGoogleLogin_Start_Bogus(c *check.C) {
resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{})
c.Check(err, check.IsNil)
c.Check(resp.RedirectLocation, check.Equals, "")
c.Check(resp.HTML.String(), check.Matches, `.*missing return_to parameter.*`)
}
-func (s *LoginSuite) TestGoogleLogin_Start(c *check.C) {
+func (s *OIDCLoginSuite) TestGoogleLogin_Start(c *check.C) {
for _, remote := range []string{"", "zzzzz"} {
resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{Remote: remote, ReturnTo: "https://app.example.com/foo?bar"})
c.Check(err, check.IsNil)
c.Check(target.Host, check.Equals, issuerURL.Host)
q := target.Query()
c.Check(q.Get("client_id"), check.Equals, "test%client$id")
- state := s.localdb.loginController.(*googleLoginController).parseOAuth2State(q.Get("state"))
+ state := s.localdb.loginController.(*oidcLoginController).parseOAuth2State(q.Get("state"))
c.Check(state.verify([]byte(s.cluster.SystemRootToken)), check.Equals, true)
c.Check(state.Time, check.Not(check.Equals), 0)
c.Check(state.Remote, check.Equals, remote)
}
}
-func (s *LoginSuite) TestGoogleLogin_InvalidCode(c *check.C) {
+func (s *OIDCLoginSuite) TestGoogleLogin_InvalidCode(c *check.C) {
state := s.startLogin(c)
resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{
Code: "first-try-a-bogus-code",
c.Check(resp.HTML.String(), check.Matches, `(?ms).*error in OAuth2 exchange.*cannot fetch token.*`)
}
-func (s *LoginSuite) TestGoogleLogin_InvalidState(c *check.C) {
+func (s *OIDCLoginSuite) TestGoogleLogin_InvalidState(c *check.C) {
s.startLogin(c)
resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{
Code: s.validCode,
c.Check(resp.HTML.String(), check.Matches, `(?ms).*invalid OAuth2 state.*`)
}
-func (s *LoginSuite) setupPeopleAPIError(c *check.C) {
+func (s *OIDCLoginSuite) setupPeopleAPIError(c *check.C) {
s.fakePeopleAPI = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
w.WriteHeader(http.StatusForbidden)
fmt.Fprintln(w, `Error 403: accessNotConfigured`)
}))
- s.localdb.loginController.(*googleLoginController).peopleAPIBasePath = s.fakePeopleAPI.URL
+ s.localdb.loginController.(*oidcLoginController).peopleAPIBasePath = s.fakePeopleAPI.URL
}
-func (s *LoginSuite) TestGoogleLogin_PeopleAPIDisabled(c *check.C) {
- s.cluster.Login.Google.AlternateEmailAddresses = false
+func (s *OIDCLoginSuite) TestGoogleLogin_PeopleAPIDisabled(c *check.C) {
+ s.localdb.loginController.(*oidcLoginController).UseGooglePeopleAPI = false
s.authEmail = "joe.smith@primary.example.com"
s.setupPeopleAPIError(c)
state := s.startLogin(c)
c.Check(authinfo.Email, check.Equals, "joe.smith@primary.example.com")
}
-func (s *LoginSuite) TestGoogleLogin_PeopleAPIError(c *check.C) {
+func (s *OIDCLoginSuite) TestConfig(c *check.C) {
+ s.cluster.Login.Google.Enable = false
+ s.cluster.Login.OpenIDConnect.Enable = true
+ s.cluster.Login.OpenIDConnect.Issuer = "https://accounts.example.com/"
+ s.cluster.Login.OpenIDConnect.ClientID = "oidc-client-id"
+ s.cluster.Login.OpenIDConnect.ClientSecret = "oidc-client-secret"
+ localdb := NewConn(s.cluster)
+ ctrl := localdb.loginController.(*oidcLoginController)
+ c.Check(ctrl.Issuer, check.Equals, "https://accounts.example.com/")
+ c.Check(ctrl.ClientID, check.Equals, "oidc-client-id")
+ c.Check(ctrl.ClientSecret, check.Equals, "oidc-client-secret")
+ c.Check(ctrl.UseGooglePeopleAPI, check.Equals, false)
+
+ for _, enableAltEmails := range []bool{false, true} {
+ s.cluster.Login.OpenIDConnect.Enable = false
+ s.cluster.Login.Google.Enable = true
+ s.cluster.Login.Google.ClientID = "google-client-id"
+ s.cluster.Login.Google.ClientSecret = "google-client-secret"
+ s.cluster.Login.Google.AlternateEmailAddresses = enableAltEmails
+ localdb = NewConn(s.cluster)
+ ctrl = localdb.loginController.(*oidcLoginController)
+ c.Check(ctrl.Issuer, check.Equals, "https://accounts.google.com")
+ c.Check(ctrl.ClientID, check.Equals, "google-client-id")
+ c.Check(ctrl.ClientSecret, check.Equals, "google-client-secret")
+ c.Check(ctrl.UseGooglePeopleAPI, check.Equals, enableAltEmails)
+ }
+}
+
+func (s *OIDCLoginSuite) TestGoogleLogin_PeopleAPIError(c *check.C) {
s.setupPeopleAPIError(c)
state := s.startLogin(c)
resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{
c.Check(resp.RedirectLocation, check.Equals, "")
}
-func (s *LoginSuite) TestGoogleLogin_Success(c *check.C) {
+func (s *OIDCLoginSuite) TestOIDCLogin_Success(c *check.C) {
+ s.cluster.Login.Google.Enable = false
+ s.cluster.Login.OpenIDConnect.Enable = true
+ json.Unmarshal([]byte(fmt.Sprintf("%q", s.fakeIssuer.URL)), &s.cluster.Login.OpenIDConnect.Issuer)
+ s.cluster.Login.OpenIDConnect.ClientID = "oidc#client#id"
+ s.cluster.Login.OpenIDConnect.ClientSecret = "oidc#client#secret"
+ s.validClientID = "oidc#client#id"
+ s.validClientSecret = "oidc#client#secret"
+ s.localdb = NewConn(s.cluster)
+ state := s.startLogin(c)
+ resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{
+ Code: s.validCode,
+ State: state,
+ })
+ c.Assert(err, check.IsNil)
+ c.Check(resp.HTML.String(), check.Equals, "")
+ target, err := url.Parse(resp.RedirectLocation)
+ c.Assert(err, check.IsNil)
+ token := target.Query().Get("api_token")
+ c.Check(token, check.Matches, `v2/zzzzz-gj3su-.{15}/.{32,50}`)
+}
+
+func (s *OIDCLoginSuite) TestGoogleLogin_Success(c *check.C) {
state := s.startLogin(c)
resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{
Code: s.validCode,
c.Check(err, check.ErrorMatches, `.*401 Unauthorized: Not logged in.*`)
}
-func (s *LoginSuite) TestGoogleLogin_RealName(c *check.C) {
+func (s *OIDCLoginSuite) TestGoogleLogin_RealName(c *check.C) {
s.authEmail = "joe.smith@primary.example.com"
s.fakePeopleAPIResponse = map[string]interface{}{
"names": []map[string]interface{}{
c.Check(authinfo.LastName, check.Equals, "Psmith")
}
-func (s *LoginSuite) TestGoogleLogin_OIDCRealName(c *check.C) {
+func (s *OIDCLoginSuite) TestGoogleLogin_OIDCRealName(c *check.C) {
s.authName = "Joe P. Smith"
s.authEmail = "joe.smith@primary.example.com"
state := s.startLogin(c)
}
// People API returns some additional email addresses.
-func (s *LoginSuite) TestGoogleLogin_AlternateEmailAddresses(c *check.C) {
+func (s *OIDCLoginSuite) TestGoogleLogin_AlternateEmailAddresses(c *check.C) {
s.authEmail = "joe.smith@primary.example.com"
s.fakePeopleAPIResponse = map[string]interface{}{
"emailAddresses": []map[string]interface{}{
}
// Primary address is not the one initially returned by oidc.
-func (s *LoginSuite) TestGoogleLogin_AlternateEmailAddresses_Primary(c *check.C) {
+func (s *OIDCLoginSuite) TestGoogleLogin_AlternateEmailAddresses_Primary(c *check.C) {
s.authEmail = "joe.smith@alternate.example.com"
s.fakePeopleAPIResponse = map[string]interface{}{
"emailAddresses": []map[string]interface{}{
c.Check(authinfo.Username, check.Equals, "jsmith")
}
-func (s *LoginSuite) TestGoogleLogin_NoPrimaryEmailAddress(c *check.C) {
+func (s *OIDCLoginSuite) TestGoogleLogin_NoPrimaryEmailAddress(c *check.C) {
s.authEmail = "joe.smith@unverified.example.com"
s.authEmailVerified = false
s.fakePeopleAPIResponse = map[string]interface{}{
c.Check(authinfo.Username, check.Equals, "")
}
-func (s *LoginSuite) startLogin(c *check.C) (state string) {
+func (s *OIDCLoginSuite) startLogin(c *check.C) (state string) {
// Initiate login, but instead of following the redirect to
// the provider, just grab state from the redirect URL.
resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{ReturnTo: "https://app.example.com/foo?bar"})
return
}
-func (s *LoginSuite) fakeToken(c *check.C, payload []byte) string {
+func (s *OIDCLoginSuite) fakeToken(c *check.C, payload []byte) string {
signer, err := jose.NewSigner(jose.SigningKey{Algorithm: jose.RS256, Key: s.issuerKey}, nil)
if err != nil {
c.Error(err)
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package undelete
+
+import (
+ "context"
+ "errors"
+ "flag"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "strings"
+ "sync"
+ "time"
+
+ "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "github.com/sirupsen/logrus"
+)
+
+var Command command
+
+type command struct{}
+
+func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ var err error
+ logger := ctxlog.New(stderr, "text", "info")
+ defer func() {
+ if err != nil {
+ logger.WithError(err).Error("fatal")
+ }
+ logger.Info("exiting")
+ }()
+
+ loader := config.NewLoader(stdin, logger)
+ loader.SkipLegacy = true
+
+ flags := flag.NewFlagSet("", flag.ContinueOnError)
+ flags.SetOutput(stderr)
+ flags.Usage = func() {
+ fmt.Fprintf(flags.Output(), `Usage:
+ %s [options ...] /path/to/manifest.txt [...]
+
+ This program recovers deleted collections. Recovery is
+ possible when the collection's manifest is still available and
+ all of its data blocks are still available or recoverable
+ (e.g., garbage collection is not enabled, the blocks are too
+ new for garbage collection, the blocks are referenced by other
+ collections, or the blocks have been trashed but not yet
+ deleted).
+
+ For each provided collection manifest, once all data blocks
+ are recovered/protected from garbage collection, a new
+ collection is saved and its UUID is printed on stdout.
+
+ Exit status will be zero if recovery is successful, i.e., a
+ collection is saved for each provided manifest.
+Options:
+`, prog)
+ flags.PrintDefaults()
+ }
+ loader.SetupFlags(flags)
+ loglevel := flags.String("log-level", "info", "logging level (debug, info, ...)")
+ err = flags.Parse(args)
+ if err == flag.ErrHelp {
+ err = nil
+ return 0
+ } else if err != nil {
+ return 2
+ }
+
+ if len(flags.Args()) == 0 {
+ flags.Usage()
+ return 2
+ }
+
+ lvl, err := logrus.ParseLevel(*loglevel)
+ if err != nil {
+ return 2
+ }
+ logger.SetLevel(lvl)
+
+ cfg, err := loader.Load()
+ if err != nil {
+ return 1
+ }
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ return 1
+ }
+ client, err := arvados.NewClientFromConfig(cluster)
+ if err != nil {
+ return 1
+ }
+ client.AuthToken = cluster.SystemRootToken
+ und := undeleter{
+ client: client,
+ cluster: cluster,
+ logger: logger,
+ }
+
+ exitcode := 0
+ for _, src := range flags.Args() {
+ logger := logger.WithField("src", src)
+ if len(src) == 27 && src[5:12] == "-57u5n-" {
+ logger.Error("log entry lookup not implemented")
+ exitcode = 1
+ continue
+ } else {
+ mtxt, err := ioutil.ReadFile(src)
+ if err != nil {
+ logger.WithError(err).Error("error loading manifest data")
+ exitcode = 1
+ continue
+ }
+ uuid, err := und.RecoverManifest(string(mtxt))
+ if err != nil {
+ logger.WithError(err).Error("recovery failed")
+ exitcode = 1
+ continue
+ }
+ logger.WithField("UUID", uuid).Info("recovery succeeded")
+ fmt.Fprintln(stdout, uuid)
+ }
+ }
+ return exitcode
+}
+
+type undeleter struct {
+ client *arvados.Client
+ cluster *arvados.Cluster
+ logger logrus.FieldLogger
+}
+
+var errNotFound = errors.New("not found")
+
+// Finds the timestamp of the newest copy of blk on svc. Returns
+// errNotFound if blk is not on svc at all.
+func (und undeleter) newestMtime(logger logrus.FieldLogger, blk string, svc arvados.KeepService) (time.Time, error) {
+ found, err := svc.Index(und.client, blk)
+ if err != nil {
+ logger.WithError(err).Warn("error getting index")
+ return time.Time{}, err
+ } else if len(found) == 0 {
+ return time.Time{}, errNotFound
+ }
+ var latest time.Time
+ for _, ent := range found {
+ t := time.Unix(0, ent.Mtime)
+ if t.After(latest) {
+ latest = t
+ }
+ }
+ logger.WithField("latest", latest).Debug("found")
+ return latest, nil
+}
+
+var errTouchIneffective = errors.New("(BUG?) touch succeeded but had no effect -- reported timestamp is still too old")
+
+// Ensures the given block exists on the given server and won't be
+// eligible for trashing until after our chosen deadline (blobsigexp).
+// Returns an error if the block doesn't exist on the given server, or
+// has an old timestamp and can't be updated.
+//
+// After we decide a block is "safe" (whether or not we had to untrash
+// it), keep-balance might notice that it's currently unreferenced and
+// decide to trash it, all before our recovered collection gets
+// saved. But if the block's timestamp is more recent than blobsigttl,
+// keepstore will refuse to trash it even if told to by keep-balance.
+func (und undeleter) ensureSafe(ctx context.Context, logger logrus.FieldLogger, blk string, svc arvados.KeepService, blobsigttl time.Duration, blobsigexp time.Time) error {
+ if latest, err := und.newestMtime(logger, blk, svc); err != nil {
+ return err
+ } else if latest.Add(blobsigttl).After(blobsigexp) {
+ return nil
+ }
+ if err := svc.Touch(ctx, und.client, blk); err != nil {
+ return fmt.Errorf("error updating timestamp: %s", err)
+ }
+ logger.Debug("updated timestamp")
+ if latest, err := und.newestMtime(logger, blk, svc); err == errNotFound {
+ return fmt.Errorf("(BUG?) touch succeeded, but then block did not appear in index")
+ } else if err != nil {
+ return err
+ } else if latest.Add(blobsigttl).After(blobsigexp) {
+ return nil
+ } else {
+ return errTouchIneffective
+ }
+}
+
+// Untrash and update GC timestamps (as needed) on blocks referenced
+// by the given manifest, save a new collection and return the new
+// collection's UUID.
+func (und undeleter) RecoverManifest(mtxt string) (string, error) {
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ coll := arvados.Collection{ManifestText: mtxt}
+ blks, err := coll.SizedDigests()
+ if err != nil {
+ return "", err
+ }
+ todo := make(chan int, len(blks))
+ for idx := range blks {
+ todo <- idx
+ }
+ go close(todo)
+
+ var services []arvados.KeepService
+ err = und.client.EachKeepService(func(svc arvados.KeepService) error {
+ if svc.ServiceType == "proxy" {
+ und.logger.WithField("service", svc).Debug("ignore proxy service")
+ } else {
+ services = append(services, svc)
+ }
+ return nil
+ })
+ if err != nil {
+ return "", fmt.Errorf("error getting list of keep services: %s", err)
+ }
+ und.logger.WithField("services", services).Debug("got list of services")
+
+ // blobsigexp is our deadline for saving the rescued
+ // collection. This must be less than BlobSigningTTL
+ // (otherwise our rescued blocks could be garbage collected
+ // again before we protect them by saving the collection) but
+ // the exact value is somewhat arbitrary. If it's too soon, it
+ // will arrive before we're ready to save, and save will
+ // fail. If it's too late, we'll needlessly update timestamps
+ // on some blocks that were recently written/touched (e.g., by
+ // a previous attempt to rescue this same collection) and
+ // would have lived long enough anyway if left alone.
+ // BlobSigningTTL/2 (typically around 1 week) is much longer
+ // than than we need to recover even a very large collection.
+ blobsigttl := und.cluster.Collections.BlobSigningTTL.Duration()
+ blobsigexp := time.Now().Add(blobsigttl / 2)
+ und.logger.WithField("blobsigexp", blobsigexp).Debug("chose save deadline")
+
+ // We'll start a number of threads, each working on
+ // checking/recovering one block at a time. The threads
+ // themselves don't need much CPU/memory, but to avoid hitting
+ // limits on keepstore connections, backend storage bandwidth,
+ // etc., we limit concurrency to 2 per keepstore node.
+ workerThreads := 2 * len(services)
+
+ blkFound := make([]bool, len(blks))
+ var wg sync.WaitGroup
+ for i := 0; i < workerThreads; i++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ nextblk:
+ for idx := range todo {
+ blk := strings.SplitN(string(blks[idx]), "+", 2)[0]
+ logger := und.logger.WithField("block", blk)
+ for _, untrashing := range []bool{false, true} {
+ for _, svc := range services {
+ logger := logger.WithField("service", fmt.Sprintf("%s:%d", svc.ServiceHost, svc.ServicePort))
+ if untrashing {
+ if err := svc.Untrash(ctx, und.client, blk); err != nil {
+ logger.WithError(err).Debug("untrash failed")
+ continue
+ }
+ logger.Info("untrashed")
+ }
+ err := und.ensureSafe(ctx, logger, blk, svc, blobsigttl, blobsigexp)
+ if err == errNotFound {
+ logger.Debug(err)
+ } else if err != nil {
+ logger.Error(err)
+ } else {
+ blkFound[idx] = true
+ continue nextblk
+ }
+ }
+ }
+ logger.Debug("unrecoverable")
+ }
+ }()
+ }
+ wg.Wait()
+
+ var have, havenot int
+ for _, ok := range blkFound {
+ if ok {
+ have++
+ } else {
+ havenot++
+ }
+ }
+ if havenot > 0 {
+ if have > 0 {
+ und.logger.Warn("partial recovery is not implemented")
+ }
+ return "", fmt.Errorf("unable to recover %d of %d blocks", havenot, have+havenot)
+ }
+
+ if und.cluster.Collections.BlobSigning {
+ key := []byte(und.cluster.Collections.BlobSigningKey)
+ coll.ManifestText = arvados.SignManifest(coll.ManifestText, und.client.AuthToken, blobsigexp, blobsigttl, key)
+ }
+ und.logger.WithField("manifest", coll.ManifestText).Debug("updated blob signatures in manifest")
+ err = und.client.RequestAndDecodeContext(ctx, &coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+ "collection": map[string]interface{}{
+ "manifest_text": coll.ManifestText,
+ },
+ })
+ if err != nil {
+ return "", fmt.Errorf("error saving new collection: %s", err)
+ }
+ und.logger.WithField("UUID", coll.UUID).Debug("created new collection")
+ return coll.UUID, nil
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package undelete
+
+import (
+ "bytes"
+ "encoding/json"
+ "io/ioutil"
+ "os"
+ "testing"
+ "time"
+
+ "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/sdk/go/arvadostest"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) {
+ check.TestingT(t)
+}
+
+var _ = check.Suite(&Suite{})
+
+type Suite struct{}
+
+func (*Suite) SetUpSuite(c *check.C) {
+ arvadostest.StartAPI()
+ arvadostest.StartKeep(2, true)
+}
+
+func (*Suite) TestUnrecoverableBlock(c *check.C) {
+ tmp := c.MkDir()
+ mfile := tmp + "/manifest"
+ ioutil.WriteFile(mfile, []byte(". aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa+410 0:410:Gone\n"), 0777)
+ var stdout, stderr bytes.Buffer
+ exitcode := Command.RunCommand("undelete.test", []string{"-log-level=debug", mfile}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 1)
+ c.Check(stdout.String(), check.Equals, "")
+ c.Log(stderr.String())
+ c.Check(stderr.String(), check.Matches, `(?ms).*msg="not found" block=aaaaa.*`)
+ c.Check(stderr.String(), check.Matches, `(?ms).*msg="untrash failed" block=aaaaa.*`)
+ c.Check(stderr.String(), check.Matches, `(?ms).*msg=unrecoverable block=aaaaa.*`)
+ c.Check(stderr.String(), check.Matches, `(?ms).*msg="recovery failed".*`)
+}
+
+func (*Suite) TestUntrashAndTouchBlock(c *check.C) {
+ tmp := c.MkDir()
+ mfile := tmp + "/manifest"
+ ioutil.WriteFile(mfile, []byte(". dcd0348cb2532ee90c99f1b846efaee7+13 0:13:test.txt\n"), 0777)
+
+ logger := ctxlog.TestLogger(c)
+ loader := config.NewLoader(&bytes.Buffer{}, logger)
+ cfg, err := loader.Load()
+ c.Assert(err, check.IsNil)
+ cluster, err := cfg.GetCluster("")
+ c.Assert(err, check.IsNil)
+ var datadirs []string
+ for _, v := range cluster.Volumes {
+ var params struct {
+ Root string
+ }
+ err := json.Unmarshal(v.DriverParameters, ¶ms)
+ c.Assert(err, check.IsNil)
+ if params.Root != "" {
+ datadirs = append(datadirs, params.Root)
+ err := os.Remove(params.Root + "/dcd/dcd0348cb2532ee90c99f1b846efaee7")
+ if err != nil && !os.IsNotExist(err) {
+ c.Error(err)
+ }
+ }
+ }
+ c.Logf("keepstore datadirs are %q", datadirs)
+
+ // Currently StartKeep(2, true) uses dirs called "keep0" and
+ // "keep1" so we could just put our fake trashed file in keep0
+ // ... but we don't want to rely on arvadostest's
+ // implementation details, so we put a trashed file in every
+ // dir that keepstore might be using.
+ for _, datadir := range datadirs {
+ if fi, err := os.Stat(datadir); err != nil || !fi.IsDir() {
+ continue
+ }
+ c.Logf("placing backdated trashed block in datadir %q", datadir)
+ trashfile := datadir + "/dcd/dcd0348cb2532ee90c99f1b846efaee7.trash.999999999"
+ os.Mkdir(datadir+"/dcd", 0777)
+ err = ioutil.WriteFile(trashfile, []byte("undelete test"), 0777)
+ c.Assert(err, check.IsNil)
+ t := time.Now().Add(-time.Hour * 24 * 365)
+ err = os.Chtimes(trashfile, t, t)
+ }
+
+ var stdout, stderr bytes.Buffer
+ exitcode := Command.RunCommand("undelete.test", []string{"-log-level=debug", mfile}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 0)
+ c.Check(stdout.String(), check.Matches, `zzzzz-4zz18-.{15}\n`)
+ c.Log(stderr.String())
+ c.Check(stderr.String(), check.Matches, `(?ms).*msg=untrashed block=dcd0348.*`)
+ c.Check(stderr.String(), check.Matches, `(?ms).*msg="updated timestamp" block=dcd0348.*`)
+
+ found := false
+ for _, datadir := range datadirs {
+ buf, err := ioutil.ReadFile(datadir + "/dcd/dcd0348cb2532ee90c99f1b846efaee7")
+ if err == nil {
+ found = true
+ c.Check(buf, check.DeepEquals, []byte("undelete test"))
+ fi, err := os.Stat(datadir + "/dcd/dcd0348cb2532ee90c99f1b846efaee7")
+ if c.Check(err, check.IsNil) {
+ c.Logf("recovered block's modtime is %s", fi.ModTime())
+ c.Check(time.Now().Sub(fi.ModTime()) < time.Hour, check.Equals, true)
+ }
+ }
+ }
+ c.Check(found, check.Equals, true)
+}
# file to determine what version of cwltool and schema-salad to
# build.
install_requires=[
- 'cwltool==3.0.20200317203547',
+ 'cwltool==3.0.20200324120055',
'schema-salad==5.0.20200302192450',
'arvados-python-client{}'.format(pysdk_dep),
'setuptools',
set -x
-if [ \$PYCMD = "python3" ]; then
+if [ "\$PYCMD" = "python3" ]; then
pip3 install cwltest
else
pip install cwltest
git clone https://github.com/common-workflow-language/cwl-\${version}.git
fi
cd cwl-\${version}
+elif [[ "$suite" != "integration" ]] ; then
+ echo "ERROR: unknown suite '$suite'"
+ exit 1
fi
if [[ "$suite" != "integration" ]] ; then
elif test "$tag" = "latest" ; then
arv-keepdocker --pull arvados/jobs $tag
else
- jobsimg=\$(curl https://versions.arvados.org/v1/commit/$tag | python -c "import json; import sys; sys.stdout.write(json.load(sys.stdin)['Versions']['Docker']['arvados/jobs'])")
- arv-keepdocker --pull arvados/jobs \$jobsimg
- docker tag arvados/jobs:\$jobsimg arvados/jobs:latest
+ set +u
+ export WORKSPACE=/usr/src/arvados
+ . /usr/src/arvados/build/run-library.sh
+ TMPHERE=\$(pwd)
+ cd /usr/src/arvados
+ calculate_python_sdk_cwl_package_versions
+ cd \$TMPHERE
+ set -u
+
+ arv-keepdocker --pull arvados/jobs \$cwl_runner_version
+ docker tag arvados/jobs:\$cwl_runner_version arvados/jobs:latest
arv-keepdocker arvados/jobs latest
fi
EXTRA=--compute-checksum
-if [[ $devcwl == 1 ]] ; then
+if [[ $devcwl -eq 1 ]] ; then
EXTRA="\$EXTRA --enable-dev"
fi
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+// Generate and verify permission signatures for Keep locators.
+//
+// See https://dev.arvados.org/projects/arvados/wiki/Keep_locator_format
+
+package arvados
+
+import (
+ "crypto/hmac"
+ "crypto/sha1"
+ "errors"
+ "fmt"
+ "regexp"
+ "strconv"
+ "strings"
+ "time"
+)
+
+var (
+ // ErrSignatureExpired - a signature was rejected because the
+ // expiry time has passed.
+ ErrSignatureExpired = errors.New("Signature expired")
+ // ErrSignatureInvalid - a signature was rejected because it
+ // was badly formatted or did not match the given secret key.
+ ErrSignatureInvalid = errors.New("Invalid signature")
+ // ErrSignatureMissing - the given locator does not have a
+ // signature hint.
+ ErrSignatureMissing = errors.New("Missing signature")
+)
+
+// makePermSignature generates a SHA-1 HMAC digest for the given blob,
+// token, expiry, and site secret.
+func makePermSignature(blobHash, apiToken, expiry, blobSignatureTTL string, permissionSecret []byte) string {
+ hmac := hmac.New(sha1.New, permissionSecret)
+ hmac.Write([]byte(blobHash))
+ hmac.Write([]byte("@"))
+ hmac.Write([]byte(apiToken))
+ hmac.Write([]byte("@"))
+ hmac.Write([]byte(expiry))
+ hmac.Write([]byte("@"))
+ hmac.Write([]byte(blobSignatureTTL))
+ digest := hmac.Sum(nil)
+ return fmt.Sprintf("%x", digest)
+}
+
+var (
+ mBlkRe = regexp.MustCompile(`^[0-9a-f]{32}.*`)
+ mPermHintRe = regexp.MustCompile(`\+A[^+]*`)
+)
+
+// SignManifest signs all locators in the given manifest, discarding
+// any existing signatures.
+func SignManifest(manifest string, apiToken string, expiry time.Time, ttl time.Duration, permissionSecret []byte) string {
+ return regexp.MustCompile(`\S+`).ReplaceAllStringFunc(manifest, func(tok string) string {
+ if mBlkRe.MatchString(tok) {
+ return SignLocator(mPermHintRe.ReplaceAllString(tok, ""), apiToken, expiry, ttl, permissionSecret)
+ } else {
+ return tok
+ }
+ })
+}
+
+// SignLocator returns blobLocator with a permission signature
+// added. If either permissionSecret or apiToken is empty, blobLocator
+// is returned untouched.
+//
+// This function is intended to be used by system components and admin
+// utilities: userland programs do not know the permissionSecret.
+func SignLocator(blobLocator, apiToken string, expiry time.Time, blobSignatureTTL time.Duration, permissionSecret []byte) string {
+ if len(permissionSecret) == 0 || apiToken == "" {
+ return blobLocator
+ }
+ // Strip off all hints: only the hash is used to sign.
+ blobHash := strings.Split(blobLocator, "+")[0]
+ timestampHex := fmt.Sprintf("%08x", expiry.Unix())
+ blobSignatureTTLHex := strconv.FormatInt(int64(blobSignatureTTL.Seconds()), 16)
+ return blobLocator +
+ "+A" + makePermSignature(blobHash, apiToken, timestampHex, blobSignatureTTLHex, permissionSecret) +
+ "@" + timestampHex
+}
+
+var SignedLocatorRe = regexp.MustCompile(
+ //1 2 34 5 6 7 89
+ `^([[:xdigit:]]{32})(\+[0-9]+)?((\+[B-Z][A-Za-z0-9@_-]*)*)(\+A([[:xdigit:]]{40})@([[:xdigit:]]{8}))((\+[B-Z][A-Za-z0-9@_-]*)*)$`)
+
+// VerifySignature returns nil if the signature on the signedLocator
+// can be verified using the given apiToken. Otherwise it returns
+// ErrSignatureExpired (if the signature's expiry time has passed,
+// which is something the client could have figured out
+// independently), ErrSignatureMissing (if there is no signature hint
+// at all), or ErrSignatureInvalid (if the signature is present but
+// badly formatted or incorrect).
+//
+// This function is intended to be used by system components and admin
+// utilities: userland programs do not know the permissionSecret.
+func VerifySignature(signedLocator, apiToken string, blobSignatureTTL time.Duration, permissionSecret []byte) error {
+ matches := SignedLocatorRe.FindStringSubmatch(signedLocator)
+ if matches == nil {
+ return ErrSignatureMissing
+ }
+ blobHash := matches[1]
+ signatureHex := matches[6]
+ expiryHex := matches[7]
+ if expiryTime, err := parseHexTimestamp(expiryHex); err != nil {
+ return ErrSignatureInvalid
+ } else if expiryTime.Before(time.Now()) {
+ return ErrSignatureExpired
+ }
+ blobSignatureTTLHex := strconv.FormatInt(int64(blobSignatureTTL.Seconds()), 16)
+ if signatureHex != makePermSignature(blobHash, apiToken, expiryHex, blobSignatureTTLHex, permissionSecret) {
+ return ErrSignatureInvalid
+ }
+ return nil
+}
+
+func parseHexTimestamp(timestampHex string) (ts time.Time, err error) {
+ if tsInt, e := strconv.ParseInt(timestampHex, 16, 0); e == nil {
+ ts = time.Unix(tsInt, 0)
+ } else {
+ err = e
+ }
+ return ts, err
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+ "time"
+
+ check "gopkg.in/check.v1"
+)
+
+const (
+ knownHash = "acbd18db4cc2f85cedef654fccc4a4d8"
+ knownLocator = knownHash + "+3"
+ knownToken = "hocfupkn2pjhrpgp2vxv8rsku7tvtx49arbc9s4bvu7p7wxqvk"
+ knownKey = "13u9fkuccnboeewr0ne3mvapk28epf68a3bhj9q8sb4l6e4e5mkk" +
+ "p6nhj2mmpscgu1zze5h5enydxfe3j215024u16ij4hjaiqs5u4pzsl3nczmaoxnc" +
+ "ljkm4875xqn4xv058koz3vkptmzhyheiy6wzevzjmdvxhvcqsvr5abhl15c2d4o4" +
+ "jhl0s91lojy1mtrzqqvprqcverls0xvy9vai9t1l1lvvazpuadafm71jl4mrwq2y" +
+ "gokee3eamvjy8qq1fvy238838enjmy5wzy2md7yvsitp5vztft6j4q866efym7e6" +
+ "vu5wm9fpnwjyxfldw3vbo01mgjs75rgo7qioh8z8ij7jpyp8508okhgbbex3ceei" +
+ "786u5rw2a9gx743dj3fgq2irk"
+ knownSignature = "89118b78732c33104a4d6231e8b5a5fa1e4301e3"
+ knownTimestamp = "7fffffff"
+ knownSigHint = "+A" + knownSignature + "@" + knownTimestamp
+ knownSignedLocator = knownLocator + knownSigHint
+ blobSignatureTTL = 1209600 * time.Second
+)
+
+var _ = check.Suite(&BlobSignatureSuite{})
+
+type BlobSignatureSuite struct{}
+
+func (s *BlobSignatureSuite) TestSignLocator(c *check.C) {
+ ts, err := parseHexTimestamp(knownTimestamp)
+ c.Check(err, check.IsNil)
+ c.Check(SignLocator(knownLocator, knownToken, ts, blobSignatureTTL, []byte(knownKey)), check.Equals, knownSignedLocator)
+}
+
+func (s *BlobSignatureSuite) TestVerifySignature(c *check.C) {
+ c.Check(VerifySignature(knownSignedLocator, knownToken, blobSignatureTTL, []byte(knownKey)), check.IsNil)
+}
+
+func (s *BlobSignatureSuite) TestVerifySignatureExtraHints(c *check.C) {
+ // handle hint before permission signature
+ c.Check(VerifySignature(knownLocator+"+K@xyzzy"+knownSigHint, knownToken, blobSignatureTTL, []byte(knownKey)), check.IsNil)
+
+ // handle hint after permission signature
+ c.Check(VerifySignature(knownLocator+knownSigHint+"+Zfoo", knownToken, blobSignatureTTL, []byte(knownKey)), check.IsNil)
+
+ // handle hints around permission signature
+ c.Check(VerifySignature(knownLocator+"+K@xyzzy"+knownSigHint+"+Zfoo", knownToken, blobSignatureTTL, []byte(knownKey)), check.IsNil)
+}
+
+// The size hint on the locator string should not affect signature
+// validation.
+func (s *BlobSignatureSuite) TestVerifySignatureWrongSize(c *check.C) {
+ // handle incorrect size hint
+ c.Check(VerifySignature(knownHash+"+999999"+knownSigHint, knownToken, blobSignatureTTL, []byte(knownKey)), check.IsNil)
+
+ // handle missing size hint
+ c.Check(VerifySignature(knownHash+knownSigHint, knownToken, blobSignatureTTL, []byte(knownKey)), check.IsNil)
+}
+
+func (s *BlobSignatureSuite) TestVerifySignatureBadSig(c *check.C) {
+ badLocator := knownLocator + "+Aaaaaaaaaaaaaaaa@" + knownTimestamp
+ c.Check(VerifySignature(badLocator, knownToken, blobSignatureTTL, []byte(knownKey)), check.Equals, ErrSignatureMissing)
+}
+
+func (s *BlobSignatureSuite) TestVerifySignatureBadTimestamp(c *check.C) {
+ badLocator := knownLocator + "+A" + knownSignature + "@OOOOOOOl"
+ c.Check(VerifySignature(badLocator, knownToken, blobSignatureTTL, []byte(knownKey)), check.Equals, ErrSignatureMissing)
+}
+
+func (s *BlobSignatureSuite) TestVerifySignatureBadSecret(c *check.C) {
+ c.Check(VerifySignature(knownSignedLocator, knownToken, blobSignatureTTL, []byte("00000000000000000000")), check.Equals, ErrSignatureInvalid)
+}
+
+func (s *BlobSignatureSuite) TestVerifySignatureBadToken(c *check.C) {
+ c.Check(VerifySignature(knownSignedLocator, "00000000", blobSignatureTTL, []byte(knownKey)), check.Equals, ErrSignatureInvalid)
+}
+
+func (s *BlobSignatureSuite) TestVerifySignatureExpired(c *check.C) {
+ yesterday := time.Now().AddDate(0, 0, -1)
+ expiredLocator := SignLocator(knownHash, knownToken, yesterday, blobSignatureTTL, []byte(knownKey))
+ c.Check(VerifySignature(expiredLocator, knownToken, blobSignatureTTL, []byte(knownKey)), check.Equals, ErrSignatureExpired)
+}
ClientSecret string
AlternateEmailAddresses bool
}
+ OpenIDConnect struct {
+ Enable bool
+ Issuer string
+ ClientID string
+ ClientSecret string
+ }
PAM struct {
Enable bool
Service string
Limit int `json:"limit"`
}
+// ContainerRequestList is an arvados#containerRequestList resource.
+type ContainerRequestList struct {
+ Items []ContainerRequest `json:"items"`
+ ItemsAvailable int `json:"items_available"`
+ Offset int `json:"offset"`
+ Limit int `json:"limit"`
+}
+
// ContainerState is a string corresponding to a valid Container state.
type ContainerState string
import (
"bufio"
+ "context"
"fmt"
+ "io/ioutil"
"net/http"
"strconv"
"strings"
return mounts, nil
}
+// Touch updates the timestamp on the given block.
+func (s *KeepService) Touch(ctx context.Context, c *Client, blk string) error {
+ req, err := http.NewRequest("TOUCH", s.url(blk), nil)
+ if err != nil {
+ return err
+ }
+ resp, err := c.Do(req.WithContext(ctx))
+ if err != nil {
+ return err
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ body, _ := ioutil.ReadAll(resp.Body)
+ return fmt.Errorf("%s %s: %s", resp.Proto, resp.Status, body)
+ }
+ return nil
+}
+
+// Untrash moves/copies the given block out of trash.
+func (s *KeepService) Untrash(ctx context.Context, c *Client, blk string) error {
+ req, err := http.NewRequest("PUT", s.url("untrash/"+blk), nil)
+ if err != nil {
+ return err
+ }
+ resp, err := c.Do(req.WithContext(ctx))
+ if err != nil {
+ return err
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ body, _ := ioutil.ReadAll(resp.Body)
+ return fmt.Errorf("%s %s: %s", resp.Proto, resp.Status, body)
+ }
+ return nil
+}
+
// Index returns an unsorted list of blocks at the given mount point.
func (s *KeepService) IndexMount(c *Client, mountUUID string, prefix string) ([]KeepServiceIndexEntry, error) {
return s.index(c, s.url("mounts/"+mountUUID+"/blocks?prefix="+prefix))
//
// SPDX-License-Identifier: Apache-2.0
-// Generate and verify permission signatures for Keep locators.
-//
-// See https://dev.arvados.org/projects/arvados/wiki/Keep_locator_format
-
package keepclient
-import (
- "crypto/hmac"
- "crypto/sha1"
- "errors"
- "fmt"
- "regexp"
- "strconv"
- "strings"
- "time"
-)
+import "git.arvados.org/arvados.git/sdk/go/arvados"
var (
- // ErrSignatureExpired - a signature was rejected because the
- // expiry time has passed.
- ErrSignatureExpired = errors.New("Signature expired")
- // ErrSignatureInvalid - a signature was rejected because it
- // was badly formatted or did not match the given secret key.
- ErrSignatureInvalid = errors.New("Invalid signature")
- // ErrSignatureMissing - the given locator does not have a
- // signature hint.
- ErrSignatureMissing = errors.New("Missing signature")
+ ErrSignatureExpired = arvados.ErrSignatureExpired
+ ErrSignatureInvalid = arvados.ErrSignatureInvalid
+ ErrSignatureMissing = arvados.ErrSignatureMissing
+ SignLocator = arvados.SignLocator
+ SignedLocatorRe = arvados.SignedLocatorRe
+ VerifySignature = arvados.VerifySignature
)
-
-// makePermSignature generates a SHA-1 HMAC digest for the given blob,
-// token, expiry, and site secret.
-func makePermSignature(blobHash, apiToken, expiry, blobSignatureTTL string, permissionSecret []byte) string {
- hmac := hmac.New(sha1.New, permissionSecret)
- hmac.Write([]byte(blobHash))
- hmac.Write([]byte("@"))
- hmac.Write([]byte(apiToken))
- hmac.Write([]byte("@"))
- hmac.Write([]byte(expiry))
- hmac.Write([]byte("@"))
- hmac.Write([]byte(blobSignatureTTL))
- digest := hmac.Sum(nil)
- return fmt.Sprintf("%x", digest)
-}
-
-// SignLocator returns blobLocator with a permission signature
-// added. If either permissionSecret or apiToken is empty, blobLocator
-// is returned untouched.
-//
-// This function is intended to be used by system components and admin
-// utilities: userland programs do not know the permissionSecret.
-func SignLocator(blobLocator, apiToken string, expiry time.Time, blobSignatureTTL time.Duration, permissionSecret []byte) string {
- if len(permissionSecret) == 0 || apiToken == "" {
- return blobLocator
- }
- // Strip off all hints: only the hash is used to sign.
- blobHash := strings.Split(blobLocator, "+")[0]
- timestampHex := fmt.Sprintf("%08x", expiry.Unix())
- blobSignatureTTLHex := strconv.FormatInt(int64(blobSignatureTTL.Seconds()), 16)
- return blobLocator +
- "+A" + makePermSignature(blobHash, apiToken, timestampHex, blobSignatureTTLHex, permissionSecret) +
- "@" + timestampHex
-}
-
-var SignedLocatorRe = regexp.MustCompile(
- //1 2 34 5 6 7 89
- `^([[:xdigit:]]{32})(\+[0-9]+)?((\+[B-Z][A-Za-z0-9@_-]*)*)(\+A([[:xdigit:]]{40})@([[:xdigit:]]{8}))((\+[B-Z][A-Za-z0-9@_-]*)*)$`)
-
-// VerifySignature returns nil if the signature on the signedLocator
-// can be verified using the given apiToken. Otherwise it returns
-// ErrSignatureExpired (if the signature's expiry time has passed,
-// which is something the client could have figured out
-// independently), ErrSignatureMissing (if there is no signature hint
-// at all), or ErrSignatureInvalid (if the signature is present but
-// badly formatted or incorrect).
-//
-// This function is intended to be used by system components and admin
-// utilities: userland programs do not know the permissionSecret.
-func VerifySignature(signedLocator, apiToken string, blobSignatureTTL time.Duration, permissionSecret []byte) error {
- matches := SignedLocatorRe.FindStringSubmatch(signedLocator)
- if matches == nil {
- return ErrSignatureMissing
- }
- blobHash := matches[1]
- signatureHex := matches[6]
- expiryHex := matches[7]
- if expiryTime, err := parseHexTimestamp(expiryHex); err != nil {
- return ErrSignatureInvalid
- } else if expiryTime.Before(time.Now()) {
- return ErrSignatureExpired
- }
- blobSignatureTTLHex := strconv.FormatInt(int64(blobSignatureTTL.Seconds()), 16)
- if signatureHex != makePermSignature(blobHash, apiToken, expiryHex, blobSignatureTTLHex, permissionSecret) {
- return ErrSignatureInvalid
- }
- return nil
-}
-
-func parseHexTimestamp(timestampHex string) (ts time.Time, err error) {
- if tsInt, e := strconv.ParseInt(timestampHex, 16, 0); e == nil {
- ts = time.Unix(tsInt, 0)
- } else {
- err = e
- }
- return ts, err
-}
+++ /dev/null
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: Apache-2.0
-
-package keepclient
-
-import (
- "testing"
- "time"
-)
-
-const (
- knownHash = "acbd18db4cc2f85cedef654fccc4a4d8"
- knownLocator = knownHash + "+3"
- knownToken = "hocfupkn2pjhrpgp2vxv8rsku7tvtx49arbc9s4bvu7p7wxqvk"
- knownKey = "13u9fkuccnboeewr0ne3mvapk28epf68a3bhj9q8sb4l6e4e5mkk" +
- "p6nhj2mmpscgu1zze5h5enydxfe3j215024u16ij4hjaiqs5u4pzsl3nczmaoxnc" +
- "ljkm4875xqn4xv058koz3vkptmzhyheiy6wzevzjmdvxhvcqsvr5abhl15c2d4o4" +
- "jhl0s91lojy1mtrzqqvprqcverls0xvy9vai9t1l1lvvazpuadafm71jl4mrwq2y" +
- "gokee3eamvjy8qq1fvy238838enjmy5wzy2md7yvsitp5vztft6j4q866efym7e6" +
- "vu5wm9fpnwjyxfldw3vbo01mgjs75rgo7qioh8z8ij7jpyp8508okhgbbex3ceei" +
- "786u5rw2a9gx743dj3fgq2irk"
- knownSignature = "89118b78732c33104a4d6231e8b5a5fa1e4301e3"
- knownTimestamp = "7fffffff"
- knownSigHint = "+A" + knownSignature + "@" + knownTimestamp
- knownSignedLocator = knownLocator + knownSigHint
- blobSignatureTTL = 1209600 * time.Second
-)
-
-func TestSignLocator(t *testing.T) {
- if ts, err := parseHexTimestamp(knownTimestamp); err != nil {
- t.Errorf("bad knownTimestamp %s", knownTimestamp)
- } else {
- if knownSignedLocator != SignLocator(knownLocator, knownToken, ts, blobSignatureTTL, []byte(knownKey)) {
- t.Fail()
- }
- }
-}
-
-func TestVerifySignature(t *testing.T) {
- if VerifySignature(knownSignedLocator, knownToken, blobSignatureTTL, []byte(knownKey)) != nil {
- t.Fail()
- }
-}
-
-func TestVerifySignatureExtraHints(t *testing.T) {
- if VerifySignature(knownLocator+"+K@xyzzy"+knownSigHint, knownToken, blobSignatureTTL, []byte(knownKey)) != nil {
- t.Fatal("Verify cannot handle hint before permission signature")
- }
-
- if VerifySignature(knownLocator+knownSigHint+"+Zfoo", knownToken, blobSignatureTTL, []byte(knownKey)) != nil {
- t.Fatal("Verify cannot handle hint after permission signature")
- }
-
- if VerifySignature(knownLocator+"+K@xyzzy"+knownSigHint+"+Zfoo", knownToken, blobSignatureTTL, []byte(knownKey)) != nil {
- t.Fatal("Verify cannot handle hints around permission signature")
- }
-}
-
-// The size hint on the locator string should not affect signature validation.
-func TestVerifySignatureWrongSize(t *testing.T) {
- if VerifySignature(knownHash+"+999999"+knownSigHint, knownToken, blobSignatureTTL, []byte(knownKey)) != nil {
- t.Fatal("Verify cannot handle incorrect size hint")
- }
-
- if VerifySignature(knownHash+knownSigHint, knownToken, blobSignatureTTL, []byte(knownKey)) != nil {
- t.Fatal("Verify cannot handle missing size hint")
- }
-}
-
-func TestVerifySignatureBadSig(t *testing.T) {
- badLocator := knownLocator + "+Aaaaaaaaaaaaaaaa@" + knownTimestamp
- if VerifySignature(badLocator, knownToken, blobSignatureTTL, []byte(knownKey)) != ErrSignatureMissing {
- t.Fail()
- }
-}
-
-func TestVerifySignatureBadTimestamp(t *testing.T) {
- badLocator := knownLocator + "+A" + knownSignature + "@OOOOOOOl"
- if VerifySignature(badLocator, knownToken, blobSignatureTTL, []byte(knownKey)) != ErrSignatureMissing {
- t.Fail()
- }
-}
-
-func TestVerifySignatureBadSecret(t *testing.T) {
- if VerifySignature(knownSignedLocator, knownToken, blobSignatureTTL, []byte("00000000000000000000")) != ErrSignatureInvalid {
- t.Fail()
- }
-}
-
-func TestVerifySignatureBadToken(t *testing.T) {
- if VerifySignature(knownSignedLocator, "00000000", blobSignatureTTL, []byte(knownKey)) != ErrSignatureInvalid {
- t.Fail()
- }
-}
-
-func TestVerifySignatureExpired(t *testing.T) {
- yesterday := time.Now().AddDate(0, 0, -1)
- expiredLocator := SignLocator(knownHash, knownToken, yesterday, blobSignatureTTL, []byte(knownKey))
- if VerifySignature(expiredLocator, knownToken, blobSignatureTTL, []byte(knownKey)) != ErrSignatureExpired {
- t.Fail()
- }
-}
'ruamel.yaml >=0.15.54, <=0.16.5',
'setuptools',
'ws4py >=0.4.2',
+ 'rsa < 4.1'
],
extras_require={
':os.name=="posix" and python_version<"3"': ['subprocess32 >= 3.5.1'],
json (>= 1.8.0)
websocket-driver (0.6.5)
websocket-extensions (>= 0.1.0)
- websocket-extensions (0.1.3)
+ websocket-extensions (0.1.5)
PLATFORMS
ruby
}
}
+// Test TOUCH requests.
+func (s *HandlerSuite) TestTouchHandler(c *check.C) {
+ c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
+ vols := s.handler.volmgr.AllWritable()
+ vols[0].Put(context.Background(), TestHash, TestBlock)
+ vols[0].Volume.(*MockVolume).TouchWithDate(TestHash, time.Now().Add(-time.Hour))
+ afterPut := time.Now()
+ t, err := vols[0].Mtime(TestHash)
+ c.Assert(err, check.IsNil)
+ c.Assert(t.Before(afterPut), check.Equals, true)
+
+ ExpectStatusCode(c,
+ "touch with no credentials",
+ http.StatusUnauthorized,
+ IssueRequest(s.handler, &RequestTester{
+ method: "TOUCH",
+ uri: "/" + TestHash,
+ }))
+
+ ExpectStatusCode(c,
+ "touch with non-root credentials",
+ http.StatusUnauthorized,
+ IssueRequest(s.handler, &RequestTester{
+ method: "TOUCH",
+ uri: "/" + TestHash,
+ apiToken: arvadostest.ActiveTokenV2,
+ }))
+
+ ExpectStatusCode(c,
+ "touch non-existent block",
+ http.StatusNotFound,
+ IssueRequest(s.handler, &RequestTester{
+ method: "TOUCH",
+ uri: "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ apiToken: s.cluster.SystemRootToken,
+ }))
+
+ beforeTouch := time.Now()
+ ExpectStatusCode(c,
+ "touch block",
+ http.StatusOK,
+ IssueRequest(s.handler, &RequestTester{
+ method: "TOUCH",
+ uri: "/" + TestHash,
+ apiToken: s.cluster.SystemRootToken,
+ }))
+ t, err = vols[0].Mtime(TestHash)
+ c.Assert(err, check.IsNil)
+ c.Assert(t.After(beforeTouch), check.Equals, true)
+}
+
// Test /index requests:
// - unauthenticated /index request
// - unauthenticated /index/prefix request
// List blocks stored here whose hash has the given prefix.
// Privileged client only.
rtr.HandleFunc(`/index/{prefix:[0-9a-f]{0,32}}`, rtr.handleIndex).Methods("GET", "HEAD")
+ // Update timestamp on existing block. Privileged client only.
+ rtr.HandleFunc(`/{hash:[0-9a-f]{32}}`, rtr.handleTOUCH).Methods("TOUCH")
// Internals/debugging info (runtime.MemStats)
rtr.HandleFunc(`/debug.json`, rtr.DebugHandler).Methods("GET", "HEAD")
}
}
+func (rtr *router) handleTOUCH(resp http.ResponseWriter, req *http.Request) {
+ if !rtr.isSystemAuth(GetAPIToken(req)) {
+ http.Error(resp, UnauthorizedError.Error(), UnauthorizedError.HTTPCode)
+ return
+ }
+ hash := mux.Vars(req)["hash"]
+ vols := rtr.volmgr.AllWritable()
+ if len(vols) == 0 {
+ http.Error(resp, "no volumes", http.StatusNotFound)
+ return
+ }
+ var err error
+ for _, mnt := range vols {
+ err = mnt.Touch(hash)
+ if err == nil {
+ break
+ }
+ }
+ switch {
+ case err == nil:
+ return
+ case os.IsNotExist(err):
+ http.Error(resp, err.Error(), http.StatusNotFound)
+ default:
+ http.Error(resp, err.Error(), http.StatusInternalServerError)
+ }
+}
+
func (rtr *router) handlePUT(resp http.ResponseWriter, req *http.Request) {
ctx, cancel := contextForResponse(context.TODO(), resp)
defer cancel()
}
func (v *MockVolume) Touch(loc string) error {
+ return v.TouchWithDate(loc, time.Now())
+}
+
+func (v *MockVolume) TouchWithDate(loc string, t time.Time) error {
v.gotCall("Touch")
<-v.Gate
if v.volume.ReadOnly {
return MethodDisabledError
}
+ if _, exists := v.Store[loc]; !exists {
+ return os.ErrNotExist
+ }
if v.Touchable {
- v.Timestamps[loc] = time.Now()
+ v.Timestamps[loc] = t
return nil
}
return errors.New("Touch failed")
TrustAllContent: true
Login:
SSO:
+ Enable: true
ProviderAppSecret: $sso_app_secret
ProviderAppID: arvados-server
Users:
popd
if [ "$PYCMD" = "python3" ]; then
- if ! pip3 install --prefix /usr/local --no-index --find-links /var/lib/pip $1 ; then
+ if ! pip3 install --prefix /usr/local --no-index --find-links /var/lib/pip $1 ; then
pip3 install --prefix /usr/local $1
- fi
+ fi
else
- if ! pip install --no-index --find-links /var/lib/pip $1 ; then
+ if ! pip install --no-index --find-links /var/lib/pip $1 ; then
pip install $1
- fi
+ fi
fi
}
GetItems() []interface{}
}
-// GroupInfo tracks previous and current members of a particular Group
+// GroupPermissions maps permission levels on groups (can_read, can_write, can_manage)
+type GroupPermissions map[string]bool
+
+// GroupInfo tracks previous and current member's permissions on a particular Group
type GroupInfo struct {
Group arvados.Group
- PreviousMembers map[string]bool
- CurrentMembers map[string]bool
+ PreviousMembers map[string]GroupPermissions
+ CurrentMembers map[string]GroupPermissions
}
// GetUserID returns the correct user id value depending on the selector
// Set up usage message
flags.Usage = func() {
- usageStr := `Synchronize remote groups into Arvados from a CSV format file with 2 columns:
- * 1st column: Group name
- * 2nd column: User identifier`
+ usageStr := `Synchronize remote groups into Arvados from a CSV format file with 3 columns:
+ * 1st: Group name
+ * 2nd: User identifier
+ * 3rd (Optional): User permission on the group: can_read, can_write or can_manage. (Default: can_write)`
fmt.Fprintf(os.Stderr, "%s\n\n", usageStr)
fmt.Fprintf(os.Stderr, "Usage:\n%s [OPTIONS] <input-file.csv>\n\n", os.Args[0])
fmt.Fprintf(os.Stderr, "Options:\n")
// Remove previous members not listed on this run
for groupUUID := range remoteGroups {
gi := remoteGroups[groupUUID]
- evictedMembers := subtract(gi.PreviousMembers, gi.CurrentMembers)
+ evictedMemberPerms := subtract(gi.PreviousMembers, gi.CurrentMembers)
groupName := gi.Group.Name
- if len(evictedMembers) > 0 {
- log.Printf("Removing %d users from group %q", len(evictedMembers), groupName)
- }
- for evictedUser := range evictedMembers {
- if err := RemoveMemberFromGroup(cfg, allUsers[userIDToUUID[evictedUser]], gi.Group); err != nil {
+ if len(evictedMemberPerms) > 0 {
+ log.Printf("Removing permissions from %d users on group %q", len(evictedMemberPerms), groupName)
+ }
+ for member := range evictedMemberPerms {
+ var perms []string
+ completeMembershipRemoval := false
+ if _, ok := gi.CurrentMembers[member]; !ok {
+ completeMembershipRemoval = true
+ membershipsRemoved++
+ } else {
+ // Collect which user->group permission links should be removed
+ for p := range evictedMemberPerms[member] {
+ if evictedMemberPerms[member][p] {
+ perms = append(perms, p)
+ }
+ }
+ membershipsRemoved += len(perms)
+ }
+ if err := RemoveMemberLinksFromGroup(cfg, allUsers[userIDToUUID[member]],
+ perms, completeMembershipRemoval, gi.Group); err != nil {
return err
}
- membershipsRemoved++
}
}
log.Printf("Groups created: %d. Memberships added: %d, removed: %d, skipped: %d", groupsCreated, membershipsAdded, membershipsRemoved, membershipsSkipped)
) (groupsCreated, membersAdded, membersSkipped int, err error) {
lineNo := 0
csvReader := csv.NewReader(f)
- csvReader.FieldsPerRecord = 2
+ // Allow variable number of fields.
+ csvReader.FieldsPerRecord = -1
for {
record, e := csvReader.Read()
if e == io.EOF {
err = fmt.Errorf("error parsing %q, line %d", cfg.Path, lineNo)
return
}
+ // Only allow 2 or 3 fields per record for backwards compatibility.
+ if len(record) < 2 || len(record) > 3 {
+ err = fmt.Errorf("error parsing %q, line %d: found %d fields but only 2 or 3 are allowed", cfg.Path, lineNo, len(record))
+ return
+ }
groupName := strings.TrimSpace(record[0])
groupMember := strings.TrimSpace(record[1]) // User ID (username or email)
- if groupName == "" || groupMember == "" {
- log.Printf("Warning: CSV record has at least one empty field (%s, %s). Skipping", groupName, groupMember)
+ groupPermission := "can_write"
+ if len(record) == 3 {
+ groupPermission = strings.ToLower(record[2])
+ }
+ if groupName == "" || groupMember == "" || groupPermission == "" {
+ log.Printf("Warning: CSV record has at least one empty field (%s, %s, %s). Skipping", groupName, groupMember, groupPermission)
+ membersSkipped++
+ continue
+ }
+ if !(groupPermission == "can_read" || groupPermission == "can_write" || groupPermission == "can_manage") {
+ log.Printf("Warning: 3rd field should be 'can_read', 'can_write' or 'can_manage'. Found: %q at line %d, skipping.", groupPermission, lineNo)
membersSkipped++
continue
}
groupNameToUUID[groupName] = newGroup.UUID
remoteGroups[newGroup.UUID] = &GroupInfo{
Group: newGroup,
- PreviousMembers: make(map[string]bool), // Empty set
- CurrentMembers: make(map[string]bool), // Empty set
+ PreviousMembers: make(map[string]GroupPermissions),
+ CurrentMembers: make(map[string]GroupPermissions),
}
groupsCreated++
}
// Both group & user exist, check if user is a member
groupUUID := groupNameToUUID[groupName]
gi := remoteGroups[groupUUID]
- if !gi.PreviousMembers[groupMember] && !gi.CurrentMembers[groupMember] {
+ if !gi.PreviousMembers[groupMember][groupPermission] && !gi.CurrentMembers[groupMember][groupPermission] {
if cfg.Verbose {
log.Printf("Adding %q to group %q", groupMember, groupName)
}
- // User wasn't a member, but should be.
- if e := AddMemberToGroup(cfg, allUsers[userIDToUUID[groupMember]], gi.Group); e != nil {
+ // User permissionwasn't there, but should be. Avoid duplicating the
+ // group->user link when necessary.
+ createG2ULink := true
+ if _, ok := gi.PreviousMembers[groupMember]; ok {
+ createG2ULink = false // User is already member of the group
+ }
+ if e := AddMemberToGroup(cfg, allUsers[userIDToUUID[groupMember]], gi.Group, groupPermission, createG2ULink); e != nil {
err = e
return
}
membersAdded++
}
- gi.CurrentMembers[groupMember] = true
+ if _, ok := gi.CurrentMembers[groupMember]; ok {
+ gi.CurrentMembers[groupMember][groupPermission] = true
+ } else {
+ gi.CurrentMembers[groupMember] = GroupPermissions{groupPermission: true}
+ }
+
}
return
}
return allItems, nil
}
-func subtract(setA map[string]bool, setB map[string]bool) map[string]bool {
- result := make(map[string]bool)
+func subtract(setA map[string]GroupPermissions, setB map[string]GroupPermissions) map[string]GroupPermissions {
+ result := make(map[string]GroupPermissions)
for element := range setA {
- if !setB[element] {
- result[element] = true
+ if _, ok := setB[element]; !ok {
+ result[element] = setA[element]
+ } else {
+ for perm := range setA[element] {
+ if _, ok := setB[element][perm]; !ok {
+ result[element] = GroupPermissions{perm: true}
+ }
+ }
}
}
return result
Operand: "permission",
}, {
Attr: "name",
- Operator: "=",
- Operand: "can_write",
+ Operator: "in",
+ Operand: []string{"can_read", "can_write", "can_manage"},
}, {
Attr: "head_uuid",
Operator: "=",
}
g2uLinks, err := GetAll(cfg.Client, "links", g2uFilter, &LinkList{})
if err != nil {
- return remoteGroups, groupNameToUUID, fmt.Errorf("error getting member (can_read) links for group %q: %s", group.Name, err)
+ return remoteGroups, groupNameToUUID, fmt.Errorf("error getting group->user 'can_read' links for group %q: %s", group.Name, err)
}
u2gLinks, err := GetAll(cfg.Client, "links", u2gFilter, &LinkList{})
if err != nil {
- return remoteGroups, groupNameToUUID, fmt.Errorf("error getting member (can_write) links for group %q: %s", group.Name, err)
+ return remoteGroups, groupNameToUUID, fmt.Errorf("error getting user->group links for group %q: %s", group.Name, err)
}
- // Build a list of user ids (email or username) belonging to this group
- membersSet := make(map[string]bool)
- u2gLinkSet := make(map[string]bool)
+ // Build a list of user ids (email or username) belonging to this group.
+ membersSet := make(map[string]GroupPermissions)
+ u2gLinkSet := make(map[string]GroupPermissions)
for _, l := range u2gLinks {
- linkedMemberUUID := l.(arvados.Link).TailUUID
- u2gLinkSet[linkedMemberUUID] = true
+ link := l.(arvados.Link)
+ // Also save the member's group access level.
+ if _, ok := u2gLinkSet[link.TailUUID]; ok {
+ u2gLinkSet[link.TailUUID][link.Name] = true
+ } else {
+ u2gLinkSet[link.TailUUID] = GroupPermissions{link.Name: true}
+ }
}
for _, item := range g2uLinks {
link := item.(arvados.Link)
if err != nil {
return remoteGroups, groupNameToUUID, err
}
- membersSet[memberID] = true
+ membersSet[memberID] = u2gLinkSet[link.HeadUUID]
}
remoteGroups[group.UUID] = &GroupInfo{
Group: group,
PreviousMembers: membersSet,
- CurrentMembers: make(map[string]bool), // Empty set
+ CurrentMembers: make(map[string]GroupPermissions),
}
groupNameToUUID[group.Name] = group.UUID
}
return remoteGroups, groupNameToUUID, nil
}
-// RemoveMemberFromGroup remove all links related to the membership
-func RemoveMemberFromGroup(cfg *ConfigParams, user arvados.User, group arvados.Group) error {
+// RemoveMemberLinksFromGroup remove all links related to the membership
+func RemoveMemberLinksFromGroup(cfg *ConfigParams, user arvados.User, linkNames []string, completeRemoval bool, group arvados.Group) error {
if cfg.Verbose {
log.Printf("Getting group membership links for user %q (%s) on group %q (%s)", user.Username, user.UUID, group.Name, group.UUID)
}
var links []interface{}
- // Search for all group<->user links (both ways)
- for _, filterset := range [][]arvados.Filter{
- // Group -> User
- {{
- Attr: "link_class",
- Operator: "=",
- Operand: "permission",
- }, {
- Attr: "tail_uuid",
- Operator: "=",
- Operand: group.UUID,
- }, {
- Attr: "head_uuid",
- Operator: "=",
- Operand: user.UUID,
- }},
- // Group <- User
- {{
- Attr: "link_class",
- Operator: "=",
- Operand: "permission",
- }, {
- Attr: "tail_uuid",
- Operator: "=",
- Operand: user.UUID,
- }, {
- Attr: "head_uuid",
- Operator: "=",
- Operand: group.UUID,
- }},
- } {
+ var filters [][]arvados.Filter
+ if completeRemoval {
+ // Search for all group<->user links (both ways)
+ filters = [][]arvados.Filter{
+ // Group -> User
+ {{
+ Attr: "link_class",
+ Operator: "=",
+ Operand: "permission",
+ }, {
+ Attr: "tail_uuid",
+ Operator: "=",
+ Operand: group.UUID,
+ }, {
+ Attr: "head_uuid",
+ Operator: "=",
+ Operand: user.UUID,
+ }},
+ // Group <- User
+ {{
+ Attr: "link_class",
+ Operator: "=",
+ Operand: "permission",
+ }, {
+ Attr: "tail_uuid",
+ Operator: "=",
+ Operand: user.UUID,
+ }, {
+ Attr: "head_uuid",
+ Operator: "=",
+ Operand: group.UUID,
+ }},
+ }
+ } else {
+ // Search only for the requested Group <- User permission links
+ filters = [][]arvados.Filter{
+ {{
+ Attr: "link_class",
+ Operator: "=",
+ Operand: "permission",
+ }, {
+ Attr: "tail_uuid",
+ Operator: "=",
+ Operand: user.UUID,
+ }, {
+ Attr: "head_uuid",
+ Operator: "=",
+ Operand: group.UUID,
+ }, {
+ Attr: "name",
+ Operator: "in",
+ Operand: linkNames,
+ }},
+ }
+ }
+
+ for _, filterset := range filters {
l, err := GetAll(cfg.Client, "links", arvados.ResourceListParams{Filters: filterset}, &LinkList{})
if err != nil {
userID, _ := GetUserID(user, cfg.UserID)
}
// AddMemberToGroup create membership links
-func AddMemberToGroup(cfg *ConfigParams, user arvados.User, group arvados.Group) error {
+func AddMemberToGroup(cfg *ConfigParams, user arvados.User, group arvados.Group, perm string, createG2ULink bool) error {
var newLink arvados.Link
- linkData := map[string]string{
- "owner_uuid": cfg.SysUserUUID,
- "link_class": "permission",
- "name": "can_read",
- "tail_uuid": group.UUID,
- "head_uuid": user.UUID,
- }
- if err := CreateLink(cfg, &newLink, linkData); err != nil {
- userID, _ := GetUserID(user, cfg.UserID)
- return fmt.Errorf("error adding group %q -> user %q read permission: %s", group.Name, userID, err)
+ var linkData map[string]string
+ if createG2ULink {
+ linkData = map[string]string{
+ "owner_uuid": cfg.SysUserUUID,
+ "link_class": "permission",
+ "name": "can_read",
+ "tail_uuid": group.UUID,
+ "head_uuid": user.UUID,
+ }
+ if err := CreateLink(cfg, &newLink, linkData); err != nil {
+ userID, _ := GetUserID(user, cfg.UserID)
+ return fmt.Errorf("error adding group %q -> user %q read permission: %s", group.Name, userID, err)
+ }
}
linkData = map[string]string{
"owner_uuid": cfg.SysUserUUID,
"link_class": "permission",
- "name": "can_write",
+ "name": perm,
"tail_uuid": user.UUID,
"head_uuid": group.UUID,
}
if err := CreateLink(cfg, &newLink, linkData); err != nil {
userID, _ := GetUserID(user, cfg.UserID)
- return fmt.Errorf("error adding user %q -> group %q write permission: %s", userID, group.Name, err)
+ return fmt.Errorf("error adding user %q -> group %q %s permission: %s", userID, group.Name, perm, err)
}
return nil
}
}
// GroupMembershipExists checks that both needed links exist between user and group
-func GroupMembershipExists(ac *arvados.Client, userUUID string, groupUUID string) bool {
+func GroupMembershipExists(ac *arvados.Client, userUUID string, groupUUID string, perm string) bool {
ll := LinkList{}
// Check Group -> User can_read permission
params := arvados.ResourceListParams{
}, {
Attr: "name",
Operator: "=",
- Operand: "can_write",
+ Operand: perm,
}, {
Attr: "tail_uuid",
Operator: "=",
groupUUID, err := RemoteGroupExists(s.cfg, groupName)
c.Assert(err, IsNil)
c.Assert(groupUUID, Not(Equals), "")
- c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID), Equals, true)
+ c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID, "can_write"), Equals, true)
}
}
+// Error out when records have <2 or >3 records
+func (s *TestSuite) TestWrongNumberOfFields(c *C) {
+ for _, testCase := range [][][]string{
+ {{"field1"}},
+ {{"field1", "field2", "field3", "field4"}},
+ {{"field1", "field2", "field3", "field4", "field5"}},
+ } {
+ tmpfile, err := MakeTempCSVFile(testCase)
+ c.Assert(err, IsNil)
+ defer os.Remove(tmpfile.Name())
+ s.cfg.Path = tmpfile.Name()
+ err = doMain(s.cfg)
+ c.Assert(err, Not(IsNil))
+ }
+}
+
+// Check different membership permissions
+func (s *TestSuite) TestMembershipLevels(c *C) {
+ userEmail := s.users[arvadostest.ActiveUserUUID].Email
+ userUUID := s.users[arvadostest.ActiveUserUUID].UUID
+ data := [][]string{
+ {"TestGroup1", userEmail, "can_read"},
+ {"TestGroup2", userEmail, "can_write"},
+ {"TestGroup3", userEmail, "can_manage"},
+ {"TestGroup4", userEmail, "invalid_permission"},
+ }
+ tmpfile, err := MakeTempCSVFile(data)
+ c.Assert(err, IsNil)
+ defer os.Remove(tmpfile.Name()) // clean up
+ s.cfg.Path = tmpfile.Name()
+ err = doMain(s.cfg)
+ c.Assert(err, IsNil)
+ for _, record := range data {
+ groupName := record[0]
+ permLevel := record[2]
+ if permLevel != "invalid_permission" {
+ groupUUID, err := RemoteGroupExists(s.cfg, groupName)
+ c.Assert(err, IsNil)
+ c.Assert(groupUUID, Not(Equals), "")
+ c.Assert(GroupMembershipExists(s.cfg.Client, userUUID, groupUUID, permLevel), Equals, true)
+ } else {
+ groupUUID, err := RemoteGroupExists(s.cfg, groupName)
+ c.Assert(err, IsNil)
+ c.Assert(groupUUID, Equals, "")
+ }
+ }
+}
+
+// Check membership level change
+func (s *TestSuite) TestMembershipLevelUpdate(c *C) {
+ userEmail := s.users[arvadostest.ActiveUserUUID].Email
+ userUUID := s.users[arvadostest.ActiveUserUUID].UUID
+ groupName := "TestGroup1"
+ // Give read permissions
+ tmpfile, err := MakeTempCSVFile([][]string{{groupName, userEmail, "can_read"}})
+ c.Assert(err, IsNil)
+ defer os.Remove(tmpfile.Name()) // clean up
+ s.cfg.Path = tmpfile.Name()
+ err = doMain(s.cfg)
+ c.Assert(err, IsNil)
+ // Check permissions
+ groupUUID, err := RemoteGroupExists(s.cfg, groupName)
+ c.Assert(err, IsNil)
+ c.Assert(groupUUID, Not(Equals), "")
+ c.Assert(GroupMembershipExists(s.cfg.Client, userUUID, groupUUID, "can_read"), Equals, true)
+ c.Assert(GroupMembershipExists(s.cfg.Client, userUUID, groupUUID, "can_write"), Equals, false)
+ c.Assert(GroupMembershipExists(s.cfg.Client, userUUID, groupUUID, "can_manage"), Equals, false)
+
+ // Give write permissions
+ tmpfile, err = MakeTempCSVFile([][]string{{groupName, userEmail, "can_write"}})
+ c.Assert(err, IsNil)
+ defer os.Remove(tmpfile.Name()) // clean up
+ s.cfg.Path = tmpfile.Name()
+ err = doMain(s.cfg)
+ c.Assert(err, IsNil)
+ // Check permissions
+ c.Assert(GroupMembershipExists(s.cfg.Client, userUUID, groupUUID, "can_read"), Equals, false)
+ c.Assert(GroupMembershipExists(s.cfg.Client, userUUID, groupUUID, "can_write"), Equals, true)
+ c.Assert(GroupMembershipExists(s.cfg.Client, userUUID, groupUUID, "can_manage"), Equals, false)
+
+ // Give manage permissions
+ tmpfile, err = MakeTempCSVFile([][]string{{groupName, userEmail, "can_manage"}})
+ c.Assert(err, IsNil)
+ defer os.Remove(tmpfile.Name()) // clean up
+ s.cfg.Path = tmpfile.Name()
+ err = doMain(s.cfg)
+ c.Assert(err, IsNil)
+ // Check permissions
+ c.Assert(GroupMembershipExists(s.cfg.Client, userUUID, groupUUID, "can_read"), Equals, false)
+ c.Assert(GroupMembershipExists(s.cfg.Client, userUUID, groupUUID, "can_write"), Equals, false)
+ c.Assert(GroupMembershipExists(s.cfg.Client, userUUID, groupUUID, "can_manage"), Equals, true)
+}
+
// The absence of a user membership on the CSV file implies its removal
func (s *TestSuite) TestMembershipRemoval(c *C) {
localUserEmail := s.users[arvadostest.ActiveUserUUID].Email
groupUUID, err := RemoteGroupExists(s.cfg, groupName)
c.Assert(err, IsNil)
c.Assert(groupUUID, Not(Equals), "")
- c.Assert(GroupMembershipExists(s.cfg.Client, localUserUUID, groupUUID), Equals, true)
- c.Assert(GroupMembershipExists(s.cfg.Client, remoteUserUUID, groupUUID), Equals, true)
+ c.Assert(GroupMembershipExists(s.cfg.Client, localUserUUID, groupUUID, "can_write"), Equals, true)
+ c.Assert(GroupMembershipExists(s.cfg.Client, remoteUserUUID, groupUUID, "can_write"), Equals, true)
}
// New CSV with some previous membership missing
data = [][]string{
groupUUID, err := RemoteGroupExists(s.cfg, "TestGroup1")
c.Assert(err, IsNil)
c.Assert(groupUUID, Not(Equals), "")
- c.Assert(GroupMembershipExists(s.cfg.Client, localUserUUID, groupUUID), Equals, true)
- c.Assert(GroupMembershipExists(s.cfg.Client, remoteUserUUID, groupUUID), Equals, false)
+ c.Assert(GroupMembershipExists(s.cfg.Client, localUserUUID, groupUUID, "can_write"), Equals, true)
+ c.Assert(GroupMembershipExists(s.cfg.Client, remoteUserUUID, groupUUID, "can_write"), Equals, false)
// Confirm TestGroup1 memberships
groupUUID, err = RemoteGroupExists(s.cfg, "TestGroup2")
c.Assert(err, IsNil)
c.Assert(groupUUID, Not(Equals), "")
- c.Assert(GroupMembershipExists(s.cfg.Client, localUserUUID, groupUUID), Equals, false)
- c.Assert(GroupMembershipExists(s.cfg.Client, remoteUserUUID, groupUUID), Equals, true)
+ c.Assert(GroupMembershipExists(s.cfg.Client, localUserUUID, groupUUID, "can_write"), Equals, false)
+ c.Assert(GroupMembershipExists(s.cfg.Client, remoteUserUUID, groupUUID, "can_write"), Equals, true)
}
// If a group doesn't exist on the system, create it before adding users
c.Assert(err, IsNil)
c.Assert(groupUUID, Not(Equals), "")
// active user should be a member
- c.Assert(GroupMembershipExists(s.cfg.Client, arvadostest.ActiveUserUUID, groupUUID), Equals, true)
+ c.Assert(GroupMembershipExists(s.cfg.Client, arvadostest.ActiveUserUUID, groupUUID, "can_write"), Equals, true)
}
// Users listed on the file that don't exist on the system are ignored
groupUUID, err = RemoteGroupExists(s.cfg, "TestGroup4")
c.Assert(err, IsNil)
c.Assert(groupUUID, Not(Equals), "")
- c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID), Equals, true)
+ c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID, "can_write"), Equals, true)
}
// Users listed on the file that don't exist on the system are ignored
activeUserEmail := s.users[arvadostest.ActiveUserUUID].Email
activeUserUUID := s.users[arvadostest.ActiveUserUUID].UUID
// Confirm that group doesn't exist
- groupUUID, err := RemoteGroupExists(s.cfg, "TestGroup4")
- c.Assert(err, IsNil)
- c.Assert(groupUUID, Equals, "")
+ for _, groupName := range []string{"TestGroup4", "TestGroup5"} {
+ groupUUID, err := RemoteGroupExists(s.cfg, groupName)
+ c.Assert(err, IsNil)
+ c.Assert(groupUUID, Equals, "")
+ }
// Create file & run command
data := [][]string{
- {"", activeUserEmail}, // Empty field
- {"TestGroup5", ""}, // Empty field
+ {"", activeUserEmail}, // Empty field
+ {"TestGroup5", ""}, // Empty field
+ {"TestGroup5", activeUserEmail, ""}, // Empty 3rd field: is optional but cannot be empty
{"TestGroup4", activeUserEmail},
}
tmpfile, err := MakeTempCSVFile(data)
s.cfg.Path = tmpfile.Name()
err = doMain(s.cfg)
c.Assert(err, IsNil)
- // Confirm that memberships exist
+ // Confirm that records about TestGroup5 were skipped
+ groupUUID, err := RemoteGroupExists(s.cfg, "TestGroup5")
+ c.Assert(err, IsNil)
+ c.Assert(groupUUID, Equals, "")
+ // Confirm that membership exists
groupUUID, err = RemoteGroupExists(s.cfg, "TestGroup4")
c.Assert(err, IsNil)
c.Assert(groupUUID, Not(Equals), "")
- c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID), Equals, true)
+ c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID, "can_write"), Equals, true)
}
// Instead of emails, use username as identifier
groupUUID, err = RemoteGroupExists(s.cfg, "TestGroup1")
c.Assert(err, IsNil)
c.Assert(groupUUID, Not(Equals), "")
- c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID), Equals, true)
+ c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID, "can_write"), Equals, true)
}
projects / arvados.git / commitdiff