fi
-debug_echo "package_go_binary $SRC_PATH"
+debug_echo "get-package-version.sh $TYPE_LANG $SRC_PATH"
if [[ "$TYPE_LANG" == "go" ]]; then
calculate_go_package_version go_package_version $SRC_PATH
NODETARBALL_=DOES_NOT_EXIST
GOVERSION=$(shell grep 'const goversion =' ../../lib/install/deps.go |awk -F'"' '{print $$2}')
GOTARBALL_x86_64=go$(GOVERSION).linux-amd64.tar.gz
-NODETARBALL_x86_64=node-v10.23.1-linux-x64.tar.xz
+NODETARBALL_x86_64=node-v12.22.12-linux-x64.tar.xz
GOTARBALL_aarch64=go$(GOVERSION).linux-arm64.tar.gz
-NODETARBALL_aarch64=node-v10.23.1-linux-arm64.tar.xz
+NODETARBALL_aarch64=node-v12.22.12-linux-arm64.tar.xz
# Get the bash variable $HOSTTYPE (this requires the SHELL line above)
HOSTTYPE=$(shell echo $${HOSTTYPE})
wget -cqO common-generated/$(GOTARBALL) https://dl.google.com/go/$(GOTARBALL)
common-generated/$(NODETARBALL): common-generated
- wget -cqO common-generated/$(NODETARBALL) https://nodejs.org/dist/v10.23.1/$(NODETARBALL)
+ wget -cqO common-generated/$(NODETARBALL) https://nodejs.org/dist/v12.22.12/$(NODETARBALL)
common-generated/$(RVMKEY1): common-generated
wget -cqO common-generated/$(RVMKEY1) https://rvm.io/mpapis.asc
ONBUILD ADD generated/go${GOVERSION}.linux-amd64.tar.gz /usr/local/
ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
# Install nodejs and npm
-ONBUILD ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/
-ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/
+ONBUILD ADD generated/node-v12.22.12-linux-x64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v12.22.12-linux-x64/bin/* /usr/local/bin/
+ONBUILD RUN npm install -g yarn
+ONBUILD RUN ln -sf /usr/local/node-v12.22.12-linux-x64/bin/* /usr/local/bin/
FROM centos:7 as build_aarch64
# Install go
ONBUILD ADD generated/go${GOVERSION}.linux-arm64.tar.gz /usr/local/
ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
# Install nodejs and npm
-ONBUILD ADD generated/node-v10.23.1-linux-arm64.tar.xz /usr/local/
-ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-arm64/bin/* /usr/local/bin/
+ONBUILD ADD generated/node-v12.22.12-linux-arm64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v12.22.12-linux-arm64/bin/* /usr/local/bin/
+ONBUILD RUN npm install -g yarn
+ONBUILD RUN ln -sf /usr/local/node-v12.22.12-linux-arm64/bin/* /usr/local/bin/
FROM build_${HOSTTYPE}
ONBUILD ADD generated/go${GOVERSION}.linux-amd64.tar.gz /usr/local/
ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
# Install nodejs and npm
-ONBUILD ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/
-ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/
+ONBUILD ADD generated/node-v12.22.12-linux-x64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v12.22.12-linux-x64/bin/* /usr/local/bin/
+ONBUILD RUN npm install -g yarn
+ONBUILD RUN ln -sf /usr/local/node-v12.22.12-linux-x64/bin/* /usr/local/bin/
# No cross compilation support for debian10 because of https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=983477
FROM debian:buster as build_aarch64
ONBUILD ADD generated/go${GOVERSION}.linux-arm64.tar.gz /usr/local/
ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
# Install nodejs and npm
-ONBUILD ADD generated/node-v10.23.1-linux-arm64.tar.xz /usr/local/
-ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-arm64/bin/* /usr/local/bin/
+ONBUILD ADD generated/node-v12.22.12-linux-arm64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v12.22.12-linux-arm64/bin/* /usr/local/bin/
+ONBUILD RUN npm install -g yarn
+ONBUILD RUN ln -sf /usr/local/node-v12.22.12-linux-arm64/bin/* /usr/local/bin/
FROM build_${HOSTTYPE}
ONBUILD ADD generated/go${GOVERSION}.linux-amd64.tar.gz /usr/local/
ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
# Install nodejs and npm
-ONBUILD ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/
-ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/
+ONBUILD ADD generated/node-v12.22.12-linux-x64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v12.22.12-linux-x64/bin/* /usr/local/bin/
+ONBUILD RUN npm install -g yarn
+ONBUILD RUN ln -sf /usr/local/node-v12.22.12-linux-x64/bin/* /usr/local/bin/
# On x86, we want some cross-compilation support for arm64
# Add gcc-aarch64-linux-gnu to compile go binaries for arm64
ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y gcc-aarch64-linux-gnu
ONBUILD ADD generated/go${GOVERSION}.linux-arm64.tar.gz /usr/local/
ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
# Install nodejs and npm
-ONBUILD ADD generated/node-v10.23.1-linux-arm64.tar.xz /usr/local/
-ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-arm64/bin/* /usr/local/bin/
+ONBUILD ADD generated/node-v12.22.12-linux-arm64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v12.22.12-linux-arm64/bin/* /usr/local/bin/
+ONBUILD RUN npm install -g yarn
+ONBUILD RUN ln -sf /usr/local/node-v12.22.12-linux-arm64/bin/* /usr/local/bin/
FROM build_${HOSTTYPE}
RUN echo HOSTTYPE ${HOSTTYPE}
ONBUILD ADD generated/go${GOVERSION}.linux-amd64.tar.gz /usr/local/
ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
# Install nodejs and npm
-ONBUILD ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/
-ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/
+ONBUILD ADD generated/node-v12.22.12-linux-x64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v12.22.12-linux-x64/bin/* /usr/local/bin/
+ONBUILD RUN npm install -g yarn
+ONBUILD RUN ln -sf /usr/local/node-v12.22.12-linux-x64/bin/* /usr/local/bin/
FROM rockylinux:8.6-minimal as build_aarch64
# Install go
ONBUILD ADD generated/go${GOVERSION}.linux-arm64.tar.gz /usr/local/
ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
# Install nodejs and npm
-ONBUILD ADD generated/node-v10.23.1-linux-arm64.tar.xz /usr/local/
-ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-arm64/bin/* /usr/local/bin/
+ONBUILD ADD generated/node-v12.22.12-linux-arm64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v12.22.12-linux-arm64/bin/* /usr/local/bin/
+ONBUILD RUN npm install -g yarn
+ONBUILD RUN ln -sf /usr/local/node-v12.22.12-linux-arm64/bin/* /usr/local/bin/
FROM build_${HOSTTYPE}
ONBUILD ADD generated/go${GOVERSION}.linux-amd64.tar.gz /usr/local/
ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
# Install nodejs and npm
-ONBUILD ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/
-ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/
+ONBUILD ADD generated/node-v12.22.12-linux-x64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v12.22.12-linux-x64/bin/* /usr/local/bin/
+ONBUILD RUN npm install -g yarn
+ONBUILD RUN ln -sf /usr/local/node-v12.22.12-linux-x64/bin/* /usr/local/bin/
# No cross compilation support for ubuntu1804 because of https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=983477
FROM ubuntu:bionic as build_aarch64
ONBUILD ADD generated/go${GOVERSION}.linux-arm64.tar.gz /usr/local/
ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
# Install nodejs and npm
-ONBUILD ADD generated/node-v10.23.1-linux-arm64.tar.xz /usr/local/
-ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-arm64/bin/* /usr/local/bin/
+ONBUILD ADD generated/node-v12.22.12-linux-arm64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v12.22.12-linux-arm64/bin/* /usr/local/bin/
+ONBUILD RUN npm install -g yarn
+ONBUILD RUN ln -sf /usr/local/node-v12.22.12-linux-arm64/bin/* /usr/local/bin/
FROM build_${HOSTTYPE}
ONBUILD ADD generated/go${GOVERSION}.linux-amd64.tar.gz /usr/local/
ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
# Install nodejs and npm
-ONBUILD ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/
-ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/
+ONBUILD ADD generated/node-v12.22.12-linux-x64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v12.22.12-linux-x64/bin/* /usr/local/bin/
+ONBUILD RUN npm install -g yarn
+ONBUILD RUN ln -sf /usr/local/node-v12.22.12-linux-x64/bin/* /usr/local/bin/
# On x86, we want some cross-compilation support for arm64
# Add gcc-aarch64-linux-gnu to compile go binaries for arm64
ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y gcc-aarch64-linux-gnu
ONBUILD ADD generated/go${GOVERSION}.linux-arm64.tar.gz /usr/local/
ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/
# Install nodejs and npm
-ONBUILD ADD generated/node-v10.23.1-linux-arm64.tar.xz /usr/local/
-ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-arm64/bin/* /usr/local/bin/
+ONBUILD ADD generated/node-v12.22.12-linux-arm64.tar.xz /usr/local/
+ONBUILD RUN ln -s /usr/local/node-v12.22.12-linux-arm64/bin/* /usr/local/bin/
+ONBUILD RUN npm install -g yarn
+ONBUILD RUN ln -sf /usr/local/node-v12.22.12-linux-arm64/bin/* /usr/local/bin/
FROM build_${HOSTTYPE}
# The cwltest package, which lives out of tree
handle_cwltest "$FORMAT" "$ARCH"
+# Workbench2
+package_workbench2
+
# Rails packages
debug_echo -e "\nRails packages\n"
fi
}
+# Usage: package_workbench2
+package_workbench2() {
+ local pkgname=arvados-workbench2
+ local src=services/workbench2
+ local dst=/var/www/arvados-workbench2/workbench2
+ local description="Arvados Workbench 2"
+ local version="$(version_from_git)"
+ cd "$WORKSPACE/$src"
+ rm -rf ./build
+ NODE_ENV=production yarn install
+ VERSION="$VERSION" BUILD_NUMBER="$(default_iteration "$pkgname" "$version" yarn)" GIT_COMMIT="$(git rev-parse HEAD | head -c9)" yarn build
+ cd "$WORKSPACE/packages/$TARGET"
+ fpm_build "${WORKSPACE}/$src" "${WORKSPACE}/$src/build/=$dst" "$pkgname" dir "$version" \
+ --license="GNU Affero General Public License, version 3.0" \
+ --description="${description}" \
+ --config-files="/etc/arvados/$pkgname/workbench2.example.json" \
+ "$WORKSPACE/services/workbench2/etc/arvados/workbench2/workbench2.example.json=/etc/arvados/$pkgname/workbench2.example.json"
+}
+
calculate_go_package_version() {
# $__returnvar has the nameref attribute set, which means it is a reference
# to another variable that is passed in as the first argument to this function.
}
# Build packages for everything
-fpm_build () {
+fpm_build() {
# Source dir where fpm-info.sh (if any) will be found.
SRC_DIR=$1
shift
FPM_RESULTS=$("${COMMAND_ARR[@]}")
FPM_EXIT_CODE=$?
+ echo "fpm: exit code $FPM_EXIT_CODE" >>$STDOUT_IF_DEBUG
+ echo "$FPM_RESULTS" >>$STDOUT_IF_DEBUG
fpm_verify $FPM_EXIT_CODE $FPM_RESULTS
FPM_RESULTS=$@
FPM_PACKAGE_NAME=''
- if [[ $FPM_RESULTS =~ ([A-Za-z0-9_\.-]*\.)(deb|rpm) ]]; then
+ if [[ $FPM_RESULTS =~ ([A-Za-z0-9_\.~-]*\.)(deb|rpm) ]]; then
FPM_PACKAGE_NAME=${BASH_REMATCH[1]}${BASH_REMATCH[2]}
fi
|| fatal "No gitolite. Try: apt-get install gitolite3"
echo -n 'npm: '
npm --version \
- || fatal "No npm. Try: wget -O- https://nodejs.org/dist/v10.23.1/node-v10.23.1-linux-x64.tar.xz | sudo tar -C /usr/local -xJf - && sudo ln -s ../node-v10.23.1-linux-x64/bin/{node,npm} /usr/local/bin/"
+ || fatal "No npm. Try: wget -O- https://nodejs.org/dist/v12.22.12/node-v12.22.12-linux-x64.tar.xz | sudo tar -C /usr/local -xJf - && sudo ln -s ../node-v12.22.12-linux-x64/bin/{node,npm} /usr/local/bin/"
echo -n 'cadaver: '
cadaver --version | grep -w cadaver \
|| fatal "No cadaver. Try: apt-get install cadaver"
}
func (conn *Conn) VocabularyGet(ctx context.Context) (arvados.Vocabulary, error) {
- return conn.chooseBackend(conn.cluster.ClusterID).VocabularyGet(ctx)
+ return conn.local.VocabularyGet(ctx)
+}
+
+func (conn *Conn) DiscoveryDocument(ctx context.Context) (arvados.DiscoveryDocument, error) {
+ return conn.local.DiscoveryDocument(ctx)
}
func (conn *Conn) Login(ctx context.Context, options arvados.LoginOptions) (arvados.LoginResponse, error) {
"first_name": true,
"is_active": true,
"is_admin": true,
+ "is_invited": true,
"last_name": true,
"modified_at": true,
"prefs": true,
"etag": false,
"full_name": false,
"identity_url": false,
- "is_invited": false,
"modified_by_client_uuid": false,
"modified_by_user_uuid": false,
"owner_uuid": false,
func (conn *Conn) batchUpdateUsers(ctx context.Context,
options arvados.ListOptions,
- items []arvados.User) (err error) {
+ items []arvados.User,
+ includeAdminAndInvited bool) (err error) {
id := conn.cluster.Login.LoginCluster
logger := ctxlog.FromContext(ctx)
}
}
}
+ if !includeAdminAndInvited {
+ // make sure we don't send these fields.
+ delete(updates, "is_admin")
+ delete(updates, "is_invited")
+ }
batchOpts.Updates[user.UUID] = updates
}
if len(batchOpts.Updates) > 0 {
return nil
}
+func (conn *Conn) includeAdminAndInvitedInBatchUpdate(ctx context.Context, be backend, updateUserUUID string) (bool, error) {
+ // API versions prior to 20231117 would only include the
+ // is_invited and is_admin fields if the current user is an
+ // admin, or is requesting their own user record. If those
+ // fields aren't actually valid then we don't want to
+ // send them in the batch update.
+ dd, err := be.DiscoveryDocument(ctx)
+ if err != nil {
+ // couldn't get discovery document
+ return false, err
+ }
+ if dd.Revision >= "20231117" {
+ // newer version, fields are valid.
+ return true, nil
+ }
+ selfuser, err := be.UserGetCurrent(ctx, arvados.GetOptions{})
+ if err != nil {
+ // couldn't get our user record
+ return false, err
+ }
+ if selfuser.IsAdmin || selfuser.UUID == updateUserUUID {
+ // we are an admin, or the current user is the same as
+ // the user that we are updating.
+ return true, nil
+ }
+ // Better safe than sorry.
+ return false, nil
+}
+
func (conn *Conn) UserList(ctx context.Context, options arvados.ListOptions) (arvados.UserList, error) {
if id := conn.cluster.Login.LoginCluster; id != "" && id != conn.cluster.ClusterID && !options.BypassFederation {
- resp, err := conn.chooseBackend(id).UserList(ctx, options)
+ be := conn.chooseBackend(id)
+ resp, err := be.UserList(ctx, options)
if err != nil {
return resp, err
}
- err = conn.batchUpdateUsers(ctx, options, resp.Items)
+ includeAdminAndInvited, err := conn.includeAdminAndInvitedInBatchUpdate(ctx, be, "")
+ if err != nil {
+ return arvados.UserList{}, err
+ }
+ err = conn.batchUpdateUsers(ctx, options, resp.Items, includeAdminAndInvited)
if err != nil {
return arvados.UserList{}, err
}
if options.BypassFederation {
return conn.local.UserUpdate(ctx, options)
}
- resp, err := conn.chooseBackend(options.UUID).UserUpdate(ctx, options)
+ be := conn.chooseBackend(options.UUID)
+ resp, err := be.UserUpdate(ctx, options)
if err != nil {
return resp, err
}
if !strings.HasPrefix(options.UUID, conn.cluster.ClusterID) {
+ includeAdminAndInvited, err := conn.includeAdminAndInvitedInBatchUpdate(ctx, be, options.UUID)
+ if err != nil {
+ return arvados.User{}, err
+ }
// Copy the updated user record to the local cluster
- err = conn.batchUpdateUsers(ctx, arvados.ListOptions{}, []arvados.User{resp})
+ err = conn.batchUpdateUsers(ctx, arvados.ListOptions{}, []arvados.User{resp}, includeAdminAndInvited)
if err != nil {
return arvados.User{}, err
}
}
func (conn *Conn) UserGet(ctx context.Context, options arvados.GetOptions) (arvados.User, error) {
- resp, err := conn.chooseBackend(options.UUID).UserGet(ctx, options)
+ be := conn.chooseBackend(options.UUID)
+ resp, err := be.UserGet(ctx, options)
if err != nil {
return resp, err
}
return arvados.User{}, httpErrorf(http.StatusBadGateway, "Had requested %v but response was for %v", options.UUID, resp.UUID)
}
if options.UUID[:5] != conn.cluster.ClusterID {
- err = conn.batchUpdateUsers(ctx, arvados.ListOptions{Select: options.Select}, []arvados.User{resp})
+ includeAdminAndInvited, err := conn.includeAdminAndInvitedInBatchUpdate(ctx, be, options.UUID)
+ if err != nil {
+ return arvados.User{}, err
+ }
+ err = conn.batchUpdateUsers(ctx, arvados.ListOptions{Select: options.Select}, []arvados.User{resp}, includeAdminAndInvited)
if err != nil {
return arvados.User{}, err
}
"identity_url": false,
// virtual attrs
"full_name": false,
- "is_invited": false,
+ "is_invited": true,
}
if opts.Select != nil {
// Only the selected
"identity_url": false,
// virtual attrs
"full_name": false,
- "is_invited": false,
+ "is_invited": true,
}
if opts.Select != nil {
// Only the selected
"net/url"
"strconv"
"strings"
+ "sync"
"time"
"git.arvados.org/arvados.git/sdk/go/arvados"
SendHeader http.Header
RedactHostInErrors bool
- clusterID string
- httpClient http.Client
- baseURL url.URL
- tokenProvider TokenProvider
+ clusterID string
+ httpClient http.Client
+ baseURL url.URL
+ tokenProvider TokenProvider
+ discoveryDocument *arvados.DiscoveryDocument
+ discoveryDocumentMtx sync.Mutex
+ discoveryDocumentExpires time.Time
}
func NewConn(clusterID string, url *url.URL, insecure bool, tp TokenProvider) *Conn {
}
if len(tokens) > 1 {
+ if params == nil {
+ params = make(map[string]interface{})
+ }
params["reader_tokens"] = tokens[1:]
}
path := ep.Path
- if strings.Contains(ep.Path, "/{uuid}") {
+ if strings.Contains(ep.Path, "/{uuid}") && params != nil {
uuid, _ := params["uuid"].(string)
path = strings.Replace(path, "/{uuid}", "/"+uuid, 1)
delete(params, "uuid")
return resp, err
}
+func (conn *Conn) DiscoveryDocument(ctx context.Context) (arvados.DiscoveryDocument, error) {
+ conn.discoveryDocumentMtx.Lock()
+ defer conn.discoveryDocumentMtx.Unlock()
+ if conn.discoveryDocument != nil && time.Now().Before(conn.discoveryDocumentExpires) {
+ return *conn.discoveryDocument, nil
+ }
+ var dd arvados.DiscoveryDocument
+ err := conn.requestAndDecode(ctx, &dd, arvados.EndpointDiscoveryDocument, nil, nil)
+ if err != nil {
+ return dd, err
+ }
+ conn.discoveryDocument = &dd
+ conn.discoveryDocumentExpires = time.Now().Add(time.Hour)
+ return *conn.discoveryDocument, nil
+}
+
func (conn *Conn) Login(ctx context.Context, options arvados.LoginOptions) (arvados.LoginResponse, error) {
ep := arvados.EndpointLogin
var resp arvados.LoginResponse
var (
EndpointConfigGet = APIEndpoint{"GET", "arvados/v1/config", ""}
EndpointVocabularyGet = APIEndpoint{"GET", "arvados/v1/vocabulary", ""}
+ EndpointDiscoveryDocument = APIEndpoint{"GET", "discovery/v1/apis/arvados/v1/rest", ""}
EndpointLogin = APIEndpoint{"GET", "login", ""}
EndpointLogout = APIEndpoint{"GET", "logout", ""}
EndpointAuthorizedKeyCreate = APIEndpoint{"POST", "arvados/v1/authorized_keys", "authorized_key"}
APIClientAuthorizationDelete(ctx context.Context, options DeleteOptions) (APIClientAuthorization, error)
APIClientAuthorizationUpdate(ctx context.Context, options UpdateOptions) (APIClientAuthorization, error)
APIClientAuthorizationGet(ctx context.Context, options GetOptions) (APIClientAuthorization, error)
+ DiscoveryDocument(ctx context.Context) (DiscoveryDocument, error)
}
GitURL string `json:"gitUrl"`
Schemas map[string]Schema `json:"schemas"`
Resources map[string]Resource `json:"resources"`
+ Revision string `json:"revision"`
}
type Resource struct {
as.appendCall(ctx, as.VocabularyGet, nil)
return arvados.Vocabulary{}, as.Error
}
+func (as *APIStub) DiscoveryDocument(ctx context.Context) (arvados.DiscoveryDocument, error) {
+ as.appendCall(ctx, as.DiscoveryDocument, nil)
+ return arvados.DiscoveryDocument{}, as.Error
+}
func (as *APIStub) Login(ctx context.Context, options arvados.LoginOptions) (arvados.LoginResponse, error) {
as.appendCall(ctx, as.Login, options)
return arvados.LoginResponse{}, as.Error
}
}
},
- "revision": "20220510",
+ "revision": "20231117",
"schemas": {
"ApiClientList": {
"id": "ApiClientList",
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
+"""Arvados Python SDK
+
+This module provides the entire Python SDK for Arvados. The most useful modules
+include:
+
+* arvados.api - After you `import arvados`, you can call `arvados.api.api` as
+ `arvados.api` to construct a client object.
+
+* arvados.collection - The `arvados.collection.Collection` class provides a
+ high-level interface to read and write collections. It coordinates sending
+ data to and from Keep, and synchronizing updates with the collection object.
+
+* arvados.util - Utility functions to use mostly in conjunction with the API
+ client object and the results it returns.
+
+Other submodules provide lower-level functionality.
+"""
-from __future__ import print_function
-from __future__ import absolute_import
-from future import standard_library
-standard_library.install_aliases()
-from builtins import object
-import bz2
-import fcntl
-import hashlib
-import http.client
-import httplib2
-import json
import logging as stdliblog
import os
-import pprint
-import re
-import string
import sys
-import time
import types
-import zlib
-if sys.version_info >= (3, 0):
- from collections import UserDict
-else:
- from UserDict import UserDict
+from collections import UserDict
from .api import api, api_from_config, http_cache
from .collection import CollectionReader, CollectionWriter, ResumableCollectionWriter
client constructors are `api` and `api_from_config`.
"""
-from __future__ import absolute_import
-from future import standard_library
-standard_library.install_aliases()
-from builtins import range
import collections
-import http.client
import httplib2
import json
import logging
import time
import types
+from typing import (
+ Any,
+ Dict,
+ List,
+ Mapping,
+ Optional,
+)
+
import apiclient
import apiclient.http
from apiclient import discovery as apiclient_discovery
errors.ApiError, *args, **kwargs)
apiclient_errors.HttpError.__new__ = staticmethod(_new_http_error)
-def http_cache(data_type):
+def http_cache(data_type: str) -> cache.SafeHTTPCache:
"""Set up an HTTP file cache
This function constructs and returns an `arvados.cache.SafeHTTPCache`
return cache.SafeHTTPCache(str(path), max_age=60*60*24*2)
def api_client(
- version,
- discoveryServiceUrl,
- token,
+ version: str,
+ discoveryServiceUrl: str,
+ token: str,
*,
- cache=True,
- http=None,
- insecure=False,
- num_retries=10,
- request_id=None,
- timeout=5*60,
- **kwargs,
-):
+ cache: bool=True,
+ http: Optional[httplib2.Http]=None,
+ insecure: bool=False,
+ num_retries: int=10,
+ request_id: Optional[str]=None,
+ timeout: int=5*60,
+ **kwargs: Any,
+) -> apiclient_discovery.Resource:
"""Build an Arvados API client
This function returns a `googleapiclient.discovery.Resource` object
Additional keyword arguments will be passed directly to
`googleapiclient.discovery.build`.
-
"""
if http is None:
http = httplib2.Http(
return svc
def normalize_api_kwargs(
- version=None,
- discoveryServiceUrl=None,
- host=None,
- token=None,
- **kwargs,
-):
+ version: Optional[str]=None,
+ discoveryServiceUrl: Optional[str]=None,
+ host: Optional[str]=None,
+ token: Optional[str]=None,
+ **kwargs: Any,
+) -> Dict[str, Any]:
"""Validate kwargs from `api` and build kwargs for `api_client`
This method takes high-level keyword arguments passed to the `api`
**kwargs,
}
-def api_kwargs_from_config(version=None, apiconfig=None, **kwargs):
+def api_kwargs_from_config(
+ version: Optional[str]=None,
+ apiconfig: Optional[Mapping[str, str]]=None,
+ **kwargs: Any
+) -> Dict[str, Any]:
"""Build `api_client` keyword arguments from configuration
This function accepts a mapping with Arvados configuration settings like
**kwargs,
)
-def api(version=None, cache=True, host=None, token=None, insecure=False,
- request_id=None, timeout=5*60, *,
- discoveryServiceUrl=None, **kwargs):
+def api(
+ version: Optional[str]=None,
+ cache: bool=True,
+ host: Optional[str]=None,
+ token: Optional[str]=None,
+ insecure: bool=False,
+ request_id: Optional[str]=None,
+ timeout: int=5*60,
+ *,
+ discoveryServiceUrl: Optional[str]=None,
+ **kwargs: Any,
+) -> 'arvados.safeapi.ThreadSafeApiCache':
"""Dynamically build an Arvados API client
This function provides a high-level "do what I mean" interface to build an
from .safeapi import ThreadSafeApiCache
return ThreadSafeApiCache({}, {}, kwargs, version)
-def api_from_config(version=None, apiconfig=None, **kwargs):
+def api_from_config(
+ version: Optional[str]=None,
+ apiconfig: Optional[Mapping[str, str]]=None,
+ **kwargs: Any
+) -> 'arvados.safeapi.ThreadSafeApiCache':
"""Build an Arvados API client from a configuration mapping
This function builds an Arvados API client from a mapping with user
#
# SPDX-License-Identifier: Apache-2.0
-from builtins import range
-from builtins import object
import functools
import inspect
import pycurl
import time
from collections import deque
+from typing import (
+ Callable,
+ Generic,
+ Optional,
+ TypeVar,
+)
import arvados.errors
_HTTP_SUCCESSES = set(range(200, 300))
_HTTP_CAN_RETRY = set([408, 409, 423, 500, 502, 503, 504])
-class RetryLoop(object):
+CT = TypeVar('CT', bound=Callable)
+T = TypeVar('T')
+
+class RetryLoop(Generic[T]):
"""Coordinate limited retries of code.
`RetryLoop` coordinates a loop that runs until it records a
it doesn't succeed. This means the loop body could run at most
`num_retries + 1` times.
- * success_check: Callable --- This is a function that will be called
- each time the loop saves a result. The function should return `True`
- if the result indicates the code succeeded, `False` if it represents a
- permanent failure, and `None` if it represents a temporary failure.
- If no function is provided, the loop will end after any result is
- saved.
+ * success_check: Callable[[T], bool | None] --- This is a function that
+ will be called each time the loop saves a result. The function should
+ return `True` if the result indicates the code succeeded, `False` if
+ it represents a permanent failure, and `None` if it represents a
+ temporary failure. If no function is provided, the loop will end
+ after any result is saved.
* backoff_start: float --- The number of seconds that must pass before
the loop's second iteration. Default 0, which disables all waiting.
* max_wait: float --- Maximum number of seconds to wait between
retries. Default 60.
"""
- def __init__(self, num_retries, success_check=lambda r: True,
- backoff_start=0, backoff_growth=2, save_results=1,
- max_wait=60):
+ def __init__(
+ self,
+ num_retries: int,
+ success_check: Callable[[T], Optional[bool]]=lambda r: True,
+ backoff_start: float=0,
+ backoff_growth: float=2,
+ save_results: int=1,
+ max_wait: float=60
+ ) -> None:
self.tries_left = num_retries + 1
self.check_result = success_check
self.backoff_wait = backoff_start
self._running = None
self._success = None
- def __iter__(self):
+ def __iter__(self) -> 'RetryLoop':
"""Return an iterator of retries."""
return self
- def running(self):
+ def running(self) -> Optional[bool]:
"""Return whether this loop is running.
Returns `None` if the loop has never run, `True` if it is still running,
"""
return self._running and (self._success is None)
- def __next__(self):
+ def __next__(self) -> int:
"""Record a loop attempt.
If the loop is still running, decrements the number of tries left and
self.tries_left -= 1
return self.tries_left
- def save_result(self, result):
+ def save_result(self, result: T) -> None:
"""Record a loop result.
Save the given result, and end the loop if it indicates
Arguments:
- * result: Any --- The result from this loop attempt to check and
- save.
+ * result: T --- The result from this loop attempt to check and save.
"""
if not self.running():
raise arvados.errors.AssertionError(
self._success = self.check_result(result)
self._attempts += 1
- def success(self):
+ def success(self) -> Optional[bool]:
"""Return the loop's end state.
Returns `True` if the loop recorded a successful result, `False` if it
"""
return self._success
- def last_result(self):
+ def last_result(self) -> T:
"""Return the most recent result the loop saved.
Raises `arvados.errors.AssertionError` if called before any result has
raise arvados.errors.AssertionError(
"queried loop results before any were recorded")
- def attempts(self):
+ def attempts(self) -> int:
"""Return the number of results that have been saved.
This count includes all kinds of results: success, permanent failure,
"""
return self._attempts
- def attempts_str(self):
+ def attempts_str(self) -> str:
"""Return a human-friendly string counting saved results.
This method returns '1 attempt' or 'N attempts', where the number
return '{} attempts'.format(self._attempts)
-def check_http_response_success(status_code):
+def check_http_response_success(status_code: int) -> Optional[bool]:
"""Convert a numeric HTTP status code to a loop control flag.
This method takes a numeric HTTP status code and returns `True` if
else:
return None # Get well soon, server.
-def retry_method(orig_func):
+def retry_method(orig_func: CT) -> CT:
"""Provide a default value for a method's num_retries argument.
This is a decorator for instance and class methods that accept a
Arvados API client.
"""
-from __future__ import absolute_import
-
-from builtins import object
import sys
import threading
+from typing import (
+ Any,
+ Mapping,
+ Optional,
+)
+
from . import config
from . import keep
from . import util
Arguments:
- apiconfig: Mapping[str, str] | None
- : A mapping with entries for `ARVADOS_API_HOST`, `ARVADOS_API_TOKEN`,
- and optionally `ARVADOS_API_HOST_INSECURE`. If not provided, uses
+ * apiconfig: Mapping[str, str] | None --- A mapping with entries for
+ `ARVADOS_API_HOST`, `ARVADOS_API_TOKEN`, and optionally
+ `ARVADOS_API_HOST_INSECURE`. If not provided, uses
`arvados.config.settings` to get these parameters from user
configuration. You can pass an empty mapping to build the client
solely from `api_params`.
- keep_params: Mapping[str, Any]
- : Keyword arguments used to construct an associated
- `arvados.keep.KeepClient`.
+ * keep_params: Mapping[str, Any] --- Keyword arguments used to construct
+ an associated `arvados.keep.KeepClient`.
- api_params: Mapping[str, Any]
- : Keyword arguments used to construct each thread's API client. These
- have the same meaning as in the `arvados.api.api` function.
+ * api_params: Mapping[str, Any] --- Keyword arguments used to construct
+ each thread's API client. These have the same meaning as in the
+ `arvados.api.api` function.
- version: str | None
- : A string naming the version of the Arvados API to use. If not specified,
- the code will log a warning and fall back to 'v1'.
+ * version: str | None --- A string naming the version of the Arvados API
+ to use. If not specified, the code will log a warning and fall back to
+ `'v1'`.
"""
-
- def __init__(self, apiconfig=None, keep_params={}, api_params={}, version=None):
+ def __init__(
+ self,
+ apiconfig: Optional[Mapping[str, str]]=None,
+ keep_params: Optional[Mapping[str, Any]]={},
+ api_params: Optional[Mapping[str, Any]]={},
+ version: Optional[str]=None,
+ ) -> None:
if apiconfig or apiconfig is None:
self._api_kwargs = api.api_kwargs_from_config(version, apiconfig, **api_params)
else:
self.local = threading.local()
self.keep = keep.KeepClient(api_client=self, **keep_params)
- def localapi(self):
+ def localapi(self) -> 'googleapiclient.discovery.Resource':
try:
client = self.local.api
except AttributeError:
self.local.api = client
return client
- def __getattr__(self, name):
+ def __getattr__(self, name: str) -> Any:
# Proxy nonexistent attributes to the thread-local API client.
return getattr(self.localapi(), name)
# Copyright (C) The Arvados Authors. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0
+"""Arvados utilities
-from __future__ import division
-from builtins import range
+This module provides functions and constants that are useful across a variety
+of Arvados resource types, or extend the Arvados API client (see `arvados.api`).
+"""
+import errno
import fcntl
import functools
import hashlib
import random
import re
import subprocess
-import errno
import sys
import warnings
import arvados.errors
+from typing import (
+ Any,
+ Callable,
+ Dict,
+ Iterator,
+ TypeVar,
+ Union,
+)
+
+T = TypeVar('T')
+
HEX_RE = re.compile(r'^[0-9a-fA-F]+$')
+"""Regular expression to match a hexadecimal string (case-insensitive)"""
CR_UNCOMMITTED = 'Uncommitted'
+"""Constant `state` value for uncommited container requests"""
CR_COMMITTED = 'Committed'
+"""Constant `state` value for committed container requests"""
CR_FINAL = 'Final'
+"""Constant `state` value for finalized container requests"""
keep_locator_pattern = re.compile(r'[0-9a-f]{32}\+[0-9]+(\+\S+)*')
+"""Regular expression to match any Keep block locator"""
signed_locator_pattern = re.compile(r'[0-9a-f]{32}\+[0-9]+(\+\S+)*\+A\S+(\+\S+)*')
+"""Regular expression to match any Keep block locator with an access token hint"""
portable_data_hash_pattern = re.compile(r'[0-9a-f]{32}\+[0-9]+')
+"""Regular expression to match any collection portable data hash"""
+manifest_pattern = re.compile(r'((\S+)( +[a-f0-9]{32}(\+[0-9]+)(\+\S+)*)+( +[0-9]+:[0-9]+:\S+)+$)+', flags=re.MULTILINE)
+"""Regular expression to match an Arvados collection manifest text"""
+keep_file_locator_pattern = re.compile(r'([0-9a-f]{32}\+[0-9]+)/(.*)')
+"""Regular expression to match a file path from a collection identified by portable data hash"""
+keepuri_pattern = re.compile(r'keep:([0-9a-f]{32}\+[0-9]+)/(.*)')
+"""Regular expression to match a `keep:` URI with a collection identified by portable data hash"""
+
uuid_pattern = re.compile(r'[a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15}')
+"""Regular expression to match any Arvados object UUID"""
collection_uuid_pattern = re.compile(r'[a-z0-9]{5}-4zz18-[a-z0-9]{15}')
+"""Regular expression to match any Arvados collection UUID"""
+container_uuid_pattern = re.compile(r'[a-z0-9]{5}-dz642-[a-z0-9]{15}')
+"""Regular expression to match any Arvados container UUID"""
group_uuid_pattern = re.compile(r'[a-z0-9]{5}-j7d0g-[a-z0-9]{15}')
-user_uuid_pattern = re.compile(r'[a-z0-9]{5}-tpzed-[a-z0-9]{15}')
+"""Regular expression to match any Arvados group UUID"""
link_uuid_pattern = re.compile(r'[a-z0-9]{5}-o0j2j-[a-z0-9]{15}')
+"""Regular expression to match any Arvados link UUID"""
+user_uuid_pattern = re.compile(r'[a-z0-9]{5}-tpzed-[a-z0-9]{15}')
+"""Regular expression to match any Arvados user UUID"""
job_uuid_pattern = re.compile(r'[a-z0-9]{5}-8i9sb-[a-z0-9]{15}')
-container_uuid_pattern = re.compile(r'[a-z0-9]{5}-dz642-[a-z0-9]{15}')
-manifest_pattern = re.compile(r'((\S+)( +[a-f0-9]{32}(\+[0-9]+)(\+\S+)*)+( +[0-9]+:[0-9]+:\S+)+$)+', flags=re.MULTILINE)
-keep_file_locator_pattern = re.compile(r'([0-9a-f]{32}\+[0-9]+)/(.*)')
-keepuri_pattern = re.compile(r'keep:([0-9a-f]{32}\+[0-9]+)/(.*)')
+"""Regular expression to match any Arvados job UUID
+
+.. WARNING:: Deprecated
+ Arvados job resources are deprecated and will be removed in a future
+ release. Prefer the containers API instead.
+"""
def _deprecated(version=None, preferred=None):
"""Mark a callable as deprecated in the SDK
If the following arguments are given, they'll be included in the
notices:
- preferred: str | None
- : The name of an alternative that users should use instead.
+ * preferred: str | None --- The name of an alternative that users should
+ use instead.
- version: str | None
- : The version of Arvados when the callable is scheduled to be
- removed.
+ * version: str | None --- The version of Arvados when the callable is
+ scheduled to be removed.
"""
if version is None:
version = ''
return deprecated_wrapper
return deprecated_decorator
+def is_hex(s: str, *length_args: int) -> bool:
+ """Indicate whether a string is a hexadecimal number
+
+ This method returns true if all characters in the string are hexadecimal
+ digits. It is case-insensitive.
+
+ You can also pass optional length arguments to check that the string has
+ the expected number of digits. If you pass one integer, the string must
+ have that length exactly, otherwise the method returns False. If you
+ pass two integers, the string's length must fall within that minimum and
+ maximum (inclusive), otherwise the method returns False.
+
+ Arguments:
+
+ * s: str --- The string to check
+
+ * length_args: int --- Optional length limit(s) for the string to check
+ """
+ num_length_args = len(length_args)
+ if num_length_args > 2:
+ raise arvados.errors.ArgumentError(
+ "is_hex accepts up to 3 arguments ({} given)".format(1 + num_length_args))
+ elif num_length_args == 2:
+ good_len = (length_args[0] <= len(s) <= length_args[1])
+ elif num_length_args == 1:
+ good_len = (len(s) == length_args[0])
+ else:
+ good_len = True
+ return bool(good_len and HEX_RE.match(s))
+
+def keyset_list_all(
+ fn: Callable[..., 'arvados.api_resources.ArvadosAPIRequest'],
+ order_key: str="created_at",
+ num_retries: int=0,
+ ascending: bool=True,
+ **kwargs: Any,
+) -> Iterator[Dict[str, Any]]:
+ """Iterate all Arvados resources from an API list call
+
+ This method takes a method that represents an Arvados API list call, and
+ iterates the objects returned by the API server. It can make multiple API
+ calls to retrieve and iterate all objects available from the API server.
+
+ Arguments:
+
+ * fn: Callable[..., arvados.api_resources.ArvadosAPIRequest] --- A
+ function that wraps an Arvados API method that returns a list of
+ objects. If you have an Arvados API client named `arv`, examples
+ include `arv.collections().list` and `arv.groups().contents`. Note
+ that you should pass the function *without* calling it.
+
+ * order_key: str --- The name of the primary object field that objects
+ should be sorted by. This name is used to build an `order` argument
+ for `fn`. Default `'created_at'`.
+
+ * num_retries: int --- This argument is passed through to
+ `arvados.api_resources.ArvadosAPIRequest.execute` for each API call. See
+ that method's docstring for details. Default 0 (meaning API calls will
+ use the `num_retries` value set when the Arvados API client was
+ constructed).
+
+ * ascending: bool --- Used to build an `order` argument for `fn`. If True,
+ all fields will be sorted in `'asc'` (ascending) order. Otherwise, all
+ fields will be sorted in `'desc'` (descending) order.
+
+ Additional keyword arguments will be passed directly to `fn` for each API
+ call. Note that this function sets `count`, `limit`, and `order` as part of
+ its work.
+ """
+ pagesize = 1000
+ kwargs["limit"] = pagesize
+ kwargs["count"] = 'none'
+ asc = "asc" if ascending else "desc"
+ kwargs["order"] = ["%s %s" % (order_key, asc), "uuid %s" % asc]
+ other_filters = kwargs.get("filters", [])
+
+ try:
+ select = set(kwargs['select'])
+ except KeyError:
+ pass
+ else:
+ select.add(order_key)
+ select.add('uuid')
+ kwargs['select'] = list(select)
+
+ nextpage = []
+ tot = 0
+ expect_full_page = True
+ seen_prevpage = set()
+ seen_thispage = set()
+ lastitem = None
+ prev_page_all_same_order_key = False
+
+ while True:
+ kwargs["filters"] = nextpage+other_filters
+ items = fn(**kwargs).execute(num_retries=num_retries)
+
+ if len(items["items"]) == 0:
+ if prev_page_all_same_order_key:
+ nextpage = [[order_key, ">" if ascending else "<", lastitem[order_key]]]
+ prev_page_all_same_order_key = False
+ continue
+ else:
+ return
+
+ seen_prevpage = seen_thispage
+ seen_thispage = set()
+
+ for i in items["items"]:
+ # In cases where there's more than one record with the
+ # same order key, the result could include records we
+ # already saw in the last page. Skip them.
+ if i["uuid"] in seen_prevpage:
+ continue
+ seen_thispage.add(i["uuid"])
+ yield i
+
+ firstitem = items["items"][0]
+ lastitem = items["items"][-1]
+
+ if firstitem[order_key] == lastitem[order_key]:
+ # Got a page where every item has the same order key.
+ # Switch to using uuid for paging.
+ nextpage = [[order_key, "=", lastitem[order_key]], ["uuid", ">" if ascending else "<", lastitem["uuid"]]]
+ prev_page_all_same_order_key = True
+ else:
+ # Start from the last order key seen, but skip the last
+ # known uuid to avoid retrieving the same row twice. If
+ # there are multiple rows with the same order key it is
+ # still likely we'll end up retrieving duplicate rows.
+ # That's handled by tracking the "seen" rows for each page
+ # so they can be skipped if they show up on the next page.
+ nextpage = [[order_key, ">=" if ascending else "<=", lastitem[order_key]], ["uuid", "!=", lastitem["uuid"]]]
+ prev_page_all_same_order_key = False
+
+def ca_certs_path(fallback: T=httplib2.CA_CERTS) -> Union[str, T]:
+ """Return the path of the best available source of CA certificates
+
+ This function checks various known paths that provide trusted CA
+ certificates, and returns the first one that exists. It checks:
+
+ * the path in the `SSL_CERT_FILE` environment variable (used by OpenSSL)
+ * `/etc/arvados/ca-certificates.crt`, respected by all Arvados software
+ * `/etc/ssl/certs/ca-certificates.crt`, the default store on Debian-based
+ distributions
+ * `/etc/pki/tls/certs/ca-bundle.crt`, the default store on Red Hat-based
+ distributions
+
+ If none of these paths exist, this function returns the value of `fallback`.
+
+ Arguments:
+
+ * fallback: T --- The value to return if none of the known paths exist.
+ The default value is the certificate store of Mozilla's trusted CAs
+ included with the Python [certifi][] package.
+
+ [certifi]: https://pypi.org/project/certifi/
+ """
+ for ca_certs_path in [
+ # SSL_CERT_FILE and SSL_CERT_DIR are openssl overrides - note
+ # that httplib2 itself also supports HTTPLIB2_CA_CERTS.
+ os.environ.get('SSL_CERT_FILE'),
+ # Arvados specific:
+ '/etc/arvados/ca-certificates.crt',
+ # Debian:
+ '/etc/ssl/certs/ca-certificates.crt',
+ # Red Hat:
+ '/etc/pki/tls/certs/ca-bundle.crt',
+ ]:
+ if ca_certs_path and os.path.exists(ca_certs_path):
+ return ca_certs_path
+ return fallback
+
+def new_request_id() -> str:
+ """Return a random request ID
+
+ This function generates and returns a random string suitable for use as a
+ `X-Request-Id` header value in the Arvados API.
+ """
+ rid = "req-"
+ # 2**104 > 36**20 > 2**103
+ n = random.getrandbits(104)
+ for _ in range(20):
+ c = n % 36
+ if c < 10:
+ rid += chr(c+ord('0'))
+ else:
+ rid += chr(c+ord('a')-10)
+ n = n // 36
+ return rid
+
+def get_config_once(svc: 'arvados.api_resources.ArvadosAPIClient') -> Dict[str, Any]:
+ """Return an Arvados cluster's configuration, with caching
+
+ This function gets and returns the Arvados configuration from the API
+ server. It caches the result on the client object and reuses it on any
+ future calls.
+
+ Arguments:
+
+ * svc: arvados.api_resources.ArvadosAPIClient --- The Arvados API client
+ object to use to retrieve and cache the Arvados cluster configuration.
+ """
+ if not svc._rootDesc.get('resources').get('configs', False):
+ # Old API server version, no config export endpoint
+ return {}
+ if not hasattr(svc, '_cached_config'):
+ svc._cached_config = svc.configs().get().execute()
+ return svc._cached_config
+
+def get_vocabulary_once(svc: 'arvados.api_resources.ArvadosAPIClient') -> Dict[str, Any]:
+ """Return an Arvados cluster's vocabulary, with caching
+
+ This function gets and returns the Arvados vocabulary from the API
+ server. It caches the result on the client object and reuses it on any
+ future calls.
+
+ .. HINT:: Low-level method
+ This is a relatively low-level wrapper around the Arvados API. Most
+ users will prefer to use `arvados.vocabulary.load_vocabulary`.
+
+ Arguments:
+
+ * svc: arvados.api_resources.ArvadosAPIClient --- The Arvados API client
+ object to use to retrieve and cache the Arvados cluster vocabulary.
+ """
+ if not svc._rootDesc.get('resources').get('vocabularies', False):
+ # Old API server version, no vocabulary export endpoint
+ return {}
+ if not hasattr(svc, '_cached_vocabulary'):
+ svc._cached_vocabulary = svc.vocabularies().get().execute()
+ return svc._cached_vocabulary
+
+def trim_name(collectionname: str) -> str:
+ """Limit the length of a name to fit within Arvados API limits
+
+ This function ensures that a string is short enough to use as an object
+ name in the Arvados API, leaving room for text that may be added by the
+ `ensure_unique_name` argument. If the source name is short enough, it is
+ returned unchanged. Otherwise, this function returns a string with excess
+ characters removed from the middle of the source string and replaced with
+ an ellipsis.
+
+ Arguments:
+
+ * collectionname: str --- The desired source name
+ """
+ max_name_len = 254 - 28
+
+ if len(collectionname) > max_name_len:
+ over = len(collectionname) - max_name_len
+ split = int(max_name_len/2)
+ collectionname = collectionname[0:split] + "…" + collectionname[split+over:]
+
+ return collectionname
+
+@_deprecated('3.0', 'arvados.util.keyset_list_all')
+def list_all(fn, num_retries=0, **kwargs):
+ # Default limit to (effectively) api server's MAX_LIMIT
+ kwargs.setdefault('limit', sys.maxsize)
+ items = []
+ offset = 0
+ items_available = sys.maxsize
+ while len(items) < items_available:
+ c = fn(offset=offset, **kwargs).execute(num_retries=num_retries)
+ items += c['items']
+ items_available = c['items_available']
+ offset = c['offset'] + len(c['items'])
+ return items
+
@_deprecated('3.0')
def clear_tmpdir(path=None):
"""
else:
allfiles += [ent_base]
return allfiles
-
-def is_hex(s, *length_args):
- """is_hex(s[, length[, max_length]]) -> boolean
-
- Return True if s is a string of hexadecimal digits.
- If one length argument is given, the string must contain exactly
- that number of digits.
- If two length arguments are given, the string must contain a number of
- digits between those two lengths, inclusive.
- Return False otherwise.
- """
- num_length_args = len(length_args)
- if num_length_args > 2:
- raise arvados.errors.ArgumentError(
- "is_hex accepts up to 3 arguments ({} given)".format(1 + num_length_args))
- elif num_length_args == 2:
- good_len = (length_args[0] <= len(s) <= length_args[1])
- elif num_length_args == 1:
- good_len = (len(s) == length_args[0])
- else:
- good_len = True
- return bool(good_len and HEX_RE.match(s))
-
-@_deprecated('3.0', 'arvados.util.keyset_list_all')
-def list_all(fn, num_retries=0, **kwargs):
- # Default limit to (effectively) api server's MAX_LIMIT
- kwargs.setdefault('limit', sys.maxsize)
- items = []
- offset = 0
- items_available = sys.maxsize
- while len(items) < items_available:
- c = fn(offset=offset, **kwargs).execute(num_retries=num_retries)
- items += c['items']
- items_available = c['items_available']
- offset = c['offset'] + len(c['items'])
- return items
-
-def keyset_list_all(fn, order_key="created_at", num_retries=0, ascending=True, **kwargs):
- pagesize = 1000
- kwargs["limit"] = pagesize
- kwargs["count"] = 'none'
- asc = "asc" if ascending else "desc"
- kwargs["order"] = ["%s %s" % (order_key, asc), "uuid %s" % asc]
- other_filters = kwargs.get("filters", [])
-
- try:
- select = set(kwargs['select'])
- except KeyError:
- pass
- else:
- select.add(order_key)
- select.add('uuid')
- kwargs['select'] = list(select)
-
- nextpage = []
- tot = 0
- expect_full_page = True
- seen_prevpage = set()
- seen_thispage = set()
- lastitem = None
- prev_page_all_same_order_key = False
-
- while True:
- kwargs["filters"] = nextpage+other_filters
- items = fn(**kwargs).execute(num_retries=num_retries)
-
- if len(items["items"]) == 0:
- if prev_page_all_same_order_key:
- nextpage = [[order_key, ">" if ascending else "<", lastitem[order_key]]]
- prev_page_all_same_order_key = False
- continue
- else:
- return
-
- seen_prevpage = seen_thispage
- seen_thispage = set()
-
- for i in items["items"]:
- # In cases where there's more than one record with the
- # same order key, the result could include records we
- # already saw in the last page. Skip them.
- if i["uuid"] in seen_prevpage:
- continue
- seen_thispage.add(i["uuid"])
- yield i
-
- firstitem = items["items"][0]
- lastitem = items["items"][-1]
-
- if firstitem[order_key] == lastitem[order_key]:
- # Got a page where every item has the same order key.
- # Switch to using uuid for paging.
- nextpage = [[order_key, "=", lastitem[order_key]], ["uuid", ">" if ascending else "<", lastitem["uuid"]]]
- prev_page_all_same_order_key = True
- else:
- # Start from the last order key seen, but skip the last
- # known uuid to avoid retrieving the same row twice. If
- # there are multiple rows with the same order key it is
- # still likely we'll end up retrieving duplicate rows.
- # That's handled by tracking the "seen" rows for each page
- # so they can be skipped if they show up on the next page.
- nextpage = [[order_key, ">=" if ascending else "<=", lastitem[order_key]], ["uuid", "!=", lastitem["uuid"]]]
- prev_page_all_same_order_key = False
-
-def ca_certs_path(fallback=httplib2.CA_CERTS):
- """Return the path of the best available CA certs source.
-
- This function searches for various distribution sources of CA
- certificates, and returns the first it finds. If it doesn't find any,
- it returns the value of `fallback` (httplib2's CA certs by default).
- """
- for ca_certs_path in [
- # SSL_CERT_FILE and SSL_CERT_DIR are openssl overrides - note
- # that httplib2 itself also supports HTTPLIB2_CA_CERTS.
- os.environ.get('SSL_CERT_FILE'),
- # Arvados specific:
- '/etc/arvados/ca-certificates.crt',
- # Debian:
- '/etc/ssl/certs/ca-certificates.crt',
- # Red Hat:
- '/etc/pki/tls/certs/ca-bundle.crt',
- ]:
- if ca_certs_path and os.path.exists(ca_certs_path):
- return ca_certs_path
- return fallback
-
-def new_request_id():
- rid = "req-"
- # 2**104 > 36**20 > 2**103
- n = random.getrandbits(104)
- for _ in range(20):
- c = n % 36
- if c < 10:
- rid += chr(c+ord('0'))
- else:
- rid += chr(c+ord('a')-10)
- n = n // 36
- return rid
-
-def get_config_once(svc):
- if not svc._rootDesc.get('resources').get('configs', False):
- # Old API server version, no config export endpoint
- return {}
- if not hasattr(svc, '_cached_config'):
- svc._cached_config = svc.configs().get().execute()
- return svc._cached_config
-
-def get_vocabulary_once(svc):
- if not svc._rootDesc.get('resources').get('vocabularies', False):
- # Old API server version, no vocabulary export endpoint
- return {}
- if not hasattr(svc, '_cached_vocabulary'):
- svc._cached_vocabulary = svc.vocabularies().get().execute()
- return svc._cached_vocabulary
-
-def trim_name(collectionname):
- """
- trim_name takes a record name (collection name, project name, etc)
- and trims it to fit the 255 character name limit, with additional
- space for the timestamp added by ensure_unique_name, by removing
- excess characters from the middle and inserting an ellipse
- """
-
- max_name_len = 254 - 28
-
- if len(collectionname) > max_name_len:
- over = len(collectionname) - max_name_len
- split = int(max_name_len/2)
- collectionname = collectionname[0:split] + "…" + collectionname[split+over:]
-
- return collectionname
#
# SPDX-License-Identifier: Apache-2.0
-from __future__ import print_function
-from __future__ import division
-from builtins import str
-from builtins import range
import argparse
import atexit
import errno
import shutil
import signal
import socket
-import string
import subprocess
import sys
import tempfile
import unittest
import yaml
-try:
- from urllib.parse import urlparse
-except ImportError:
- from urlparse import urlparse
+from urllib.parse import urlparse
MY_DIRNAME = os.path.dirname(os.path.realpath(__file__))
if __name__ == '__main__' and os.path.exists(
# format is YYYYMMDD, must be fixed width (needs to be lexically
# sortable), updated manually, may be used by clients to
# determine availability of API server features.
- revision: "20220510",
+ revision: "20231117",
source_version: AppVersion.hash,
sourceVersion: AppVersion.hash, # source_version should be deprecated in the future
packageVersion: AppVersion.package_version,
# records from LoginCluster.
def batch_update
@objects = []
- params[:updates].andand.each do |uuid, attrs|
- begin
- u = User.find_or_create_by(uuid: uuid)
- rescue ActiveRecord::RecordNotUnique
- retry
- end
- needupdate = {}
- nullify_attrs(attrs).each do |k,v|
- if !v.nil? && u.send(k) != v
- needupdate[k] = v
- end
- end
- if needupdate.length > 0
- begin
- u.update!(needupdate)
- rescue ActiveRecord::RecordInvalid
- loginCluster = Rails.configuration.Login.LoginCluster
- if u.uuid[0..4] == loginCluster && !needupdate[:username].nil?
- local_user = User.find_by_username(needupdate[:username])
- # The username of this record conflicts with an existing,
- # different user record. This can happen because the
- # username changed upstream on the login cluster, or
- # because we're federated with another cluster with a user
- # by the same username. The login cluster is the source
- # of truth, so change the username on the conflicting
- # record and retry the update operation.
- if local_user.uuid != u.uuid
- new_username = "#{needupdate[:username]}#{rand(99999999)}"
- Rails.logger.warn("cached username '#{needupdate[:username]}' collision with user '#{local_user.uuid}' - renaming to '#{new_username}' before retrying")
- local_user.update!({username: new_username})
- retry
- end
- end
- raise # Not the issue we're handling above
- end
- end
+ # update_remote_user takes a row lock on the User record, so sort
+ # the keys so we always lock them in the same order.
+ sorted = params[:updates].keys.sort
+ sorted.each do |uuid|
+ attrs = params[:updates][uuid]
+ attrs[:uuid] = uuid
+ u = User.update_remote_user nullify_attrs(attrs)
@objects << u
end
@offset = 0
return super if @read_users.any?(&:is_admin)
if params[:uuid] != current_user.andand.uuid
# Non-admin index/show returns very basic information about readable users.
- safe_attrs = ["uuid", "is_active", "email", "first_name", "last_name", "username", "can_write", "can_manage", "kind"]
+ safe_attrs = ["uuid", "is_active", "is_admin", "is_invited", "email", "first_name", "last_name", "username", "can_write", "can_manage", "kind"]
if @select
@select = @select & safe_attrs
else
if user.nil? and remote_user.nil?
Rails.logger.warn "remote token #{token.inspect} rejected: cannot get owner #{remote_user_uuid} from database or remote cluster"
return nil
+ end
+
# Invariant: remote_user_prefix == upstream_cluster_id
# therefore: remote_user_prefix != Rails.configuration.ClusterID
# Add or update user and token in local database so we can
# validate subsequent requests faster.
- elsif user.nil?
- # Create a new record for this user.
- user = User.new(uuid: remote_user['uuid'],
- is_active: false,
- is_admin: false,
- email: remote_user['email'],
- owner_uuid: system_user_uuid)
- user.set_initial_username(requested: remote_user['username'])
- end
- # Sync user record if we loaded a remote user.
act_as_system_user do
- if remote_user
- %w[first_name last_name email prefs].each do |attr|
- user.send(attr+'=', remote_user[attr])
- end
-
- begin
- user.save!
- rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotUnique
- Rails.logger.debug("remote user #{remote_user['uuid']} already exists, retrying...")
- # Some other request won the race: retry fetching the user record.
- user = User.uncached do
- User.find_by_uuid(remote_user['uuid'])
- end
- if !user
- Rails.logger.warn("cannot find or create remote user #{remote_user['uuid']}")
- return nil
- end
- end
-
- if user.is_invited && !remote_user['is_invited']
- # Remote user is not "invited" state, they should be unsetup, which
- # also makes them inactive.
- user.unsetup
- else
- if !user.is_invited && remote_user['is_invited'] and
- (remote_user_prefix == Rails.configuration.Login.LoginCluster or
- Rails.configuration.Users.AutoSetupNewUsers or
- Rails.configuration.Users.NewUsersAreActive or
- Rails.configuration.RemoteClusters[remote_user_prefix].andand["ActivateUsers"])
- user.setup
- end
-
- if !user.is_active && remote_user['is_active'] && user.is_invited and
- (remote_user_prefix == Rails.configuration.Login.LoginCluster or
- Rails.configuration.Users.NewUsersAreActive or
- Rails.configuration.RemoteClusters[remote_user_prefix].andand["ActivateUsers"])
- user.update!(is_active: true)
- elsif user.is_active && !remote_user['is_active']
- user.update!(is_active: false)
- end
-
- if remote_user_prefix == Rails.configuration.Login.LoginCluster and
- user.is_active and
- user.is_admin != remote_user['is_admin']
- # Remote cluster controls our user database, including the
- # admin flag.
- user.update!(is_admin: remote_user['is_admin'])
- end
- end
+ if remote_user && remote_user_uuid != anonymous_user_uuid
+ # Sync user record if we loaded a remote user.
+ user = User.update_remote_user remote_user
end
# If stored_secret is set, we save stored_secret in the database
before_destroy :ensure_owner_uuid_is_permitted
before_destroy :ensure_permission_to_destroy
before_create :update_modified_by_fields
+ before_create :add_uuid_to_name, :if => Proc.new { @_add_uuid_to_name }
before_update :maybe_update_modified_by_fields
after_create :log_create
after_update :log_update
end
def save_with_unique_name!
- uuid_was = uuid
- name_was = name
max_retries = 2
transaction do
conn = ActiveRecord::Base.connection
conn.exec_query 'ROLLBACK TO SAVEPOINT save_with_unique_name'
- new_name = "#{name_was} (#{db_current_time.utc.iso8601(3)})"
- if new_name == name
- # If the database is fast enough to do two attempts in the
- # same millisecond, we need to wait to ensure we try a
- # different timestamp on each attempt.
- sleep 0.002
- new_name = "#{name_was} (#{db_current_time.utc.iso8601(3)})"
- end
-
- self[:name] = new_name
- if uuid_was.nil? && !uuid.nil?
+ if uuid_was.nil?
+ # new record, the uuid caused a name collision (very
+ # unlikely but possible), so generate new uuid
self[:uuid] = nil
if self.is_a? Collection
- # Reset so that is assigned to the new UUID
+ # Also needs to be reset
self[:current_version_uuid] = nil
end
+ # need to adjust the name after the uuid has been generated
+ add_uuid_to_make_unique_name
+ else
+ # existing record, just update the name directly.
+ add_uuid_to_name
end
-
retry
end
end
*ft[:param_out])
end
+ @_add_uuid_to_name = false
+ def add_uuid_to_make_unique_name
+ @_add_uuid_to_name = true
+ end
+
+ def add_uuid_to_name
+ # Incorporate the random part of the UUID into the name. This
+ # lets us prevent name collision but the part we add to the name
+ # is still somewhat meaningful (instead of generating a second
+ # random meaningless string).
+ #
+ # Because ArvadosModel is an abstract class and assign_uuid is
+ # part of HasUuid (which is included by the other concrete
+ # classes) the assign_uuid hook gets added (and run) after this
+ # one. So we need to call assign_uuid here to make sure we have a
+ # uuid.
+ assign_uuid
+ self.name = "#{self.name[0..236]} (#{self.uuid[-15..-1]})"
+ end
+
protected
def self.deep_sort_hash(x)
before_create :set_initial_username, :if => Proc.new {
username.nil? and email
}
+ before_create :active_is_not_nil
after_create :after_ownership_change
after_create :setup_on_activate
after_create :add_system_group_permission_link
self.groups_i_can(:read).select { |x| x.match(/-f+$/) }.first)
end
+ def self.ignored_select_attributes
+ super + ["full_name", "is_invited"]
+ end
+
def groups_i_can(verb)
my_groups = self.group_permissions(VAL_FOR_PERM[verb]).keys
if verb == :read
end
def set_initial_username(requested: false)
- if !requested.is_a?(String) || requested.empty?
+ if (!requested.is_a?(String) || requested.empty?) and email
email_parts = email.partition("@")
local_parts = email_parts.first.partition("+")
if email_parts.any?(&:empty?)
requested = email_parts.first
end
end
- requested.sub!(/^[^A-Za-z]+/, "")
- requested.gsub!(/[^A-Za-z0-9]/, "")
- unless requested.empty?
+ if requested
+ requested.sub!(/^[^A-Za-z]+/, "")
+ requested.gsub!(/[^A-Za-z0-9]/, "")
+ end
+ unless !requested || requested.empty?
self.username = find_usable_username_from(requested)
end
end
+ def active_is_not_nil
+ self.is_active = false if self.is_active.nil?
+ self.is_admin = false if self.is_admin.nil?
+ end
+
# Move this user's (i.e., self's) owned items to new_owner_uuid and
# new_user_uuid (for things normally owned directly by the user).
#
primary_user
end
+ def self.update_remote_user remote_user
+ remote_user = remote_user.symbolize_keys
+ begin
+ user = User.find_or_create_by(uuid: remote_user[:uuid])
+ rescue ActiveRecord::RecordNotUnique
+ retry
+ end
+
+ remote_user_prefix = user.uuid[0..4]
+ user.with_lock do
+ needupdate = {}
+ [:email, :username, :first_name, :last_name, :prefs].each do |k|
+ v = remote_user[k]
+ if !v.nil? && user.send(k) != v
+ needupdate[k] = v
+ end
+ end
+
+ user.email = needupdate[:email] if needupdate[:email]
+
+ loginCluster = Rails.configuration.Login.LoginCluster
+ if user.username.nil? || user.username == ""
+ # Don't have a username yet, set one
+ needupdate[:username] = user.set_initial_username(requested: remote_user[:username])
+ elsif remote_user_prefix != loginCluster
+ # Upstream is not login cluster, don't try to change the
+ # username once set.
+ needupdate.delete :username
+ end
+
+ if needupdate.length > 0
+ begin
+ user.update!(needupdate)
+ rescue ActiveRecord::RecordInvalid
+ if remote_user_prefix == loginCluster && !needupdate[:username].nil?
+ local_user = User.find_by_username(needupdate[:username])
+ # The username of this record conflicts with an existing,
+ # different user record. This can happen because the
+ # username changed upstream on the login cluster, or
+ # because we're federated with another cluster with a user
+ # by the same username. The login cluster is the source
+ # of truth, so change the username on the conflicting
+ # record and retry the update operation.
+ if local_user.uuid != user.uuid
+ new_username = "#{needupdate[:username]}#{rand(99999999)}"
+ Rails.logger.warn("cached username '#{needupdate[:username]}' collision with user '#{local_user.uuid}' - renaming to '#{new_username}' before retrying")
+ local_user.update!({username: new_username})
+ retry
+ end
+ end
+ raise # Not the issue we're handling above
+ end
+ end
+
+ if user.is_invited && remote_user[:is_invited] == false
+ # Remote user is not "invited" state, they should be unsetup, which
+ # also makes them inactive.
+ user.unsetup
+ else
+ if !user.is_invited && remote_user[:is_invited] and
+ (remote_user_prefix == Rails.configuration.Login.LoginCluster or
+ Rails.configuration.Users.AutoSetupNewUsers or
+ Rails.configuration.Users.NewUsersAreActive or
+ Rails.configuration.RemoteClusters[remote_user_prefix].andand["ActivateUsers"])
+ # Remote user is 'invited' and should be set up
+ user.setup
+ end
+
+ if !user.is_active && remote_user[:is_active] && user.is_invited and
+ (remote_user_prefix == Rails.configuration.Login.LoginCluster or
+ Rails.configuration.Users.NewUsersAreActive or
+ Rails.configuration.RemoteClusters[remote_user_prefix].andand["ActivateUsers"])
+ # remote user is active and invited, we need to activate them
+ user.update!(is_active: true)
+ elsif user.is_active && remote_user[:is_active] == false
+ # remote user is not active, we need to de-activate them
+ user.update!(is_active: false)
+ end
+
+ if remote_user_prefix == Rails.configuration.Login.LoginCluster and
+ user.is_active and
+ !remote_user[:is_admin].nil? and
+ user.is_admin != remote_user[:is_admin]
+ # Remote cluster controls our user database, including the
+ # admin flag.
+ user.update!(is_admin: remote_user[:is_admin])
+ end
+ end
+ end
+ user
+ end
+
protected
def self.attributes_required_columns
name: can_read
head_uuid: zzzzz-j7d0g-publicfavorites
properties: {}
+
+future_project_user_member_of_all_users_group:
+ uuid: zzzzz-o0j2j-cdnq6627g0h0r2a
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2015-07-28T21:34:41.361747000Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2015-07-28T21:34:41.361747000Z
+ updated_at: 2015-07-28T21:34:41.361747000Z
+ tail_uuid: zzzzz-tpzed-futureprojview2
+ link_class: permission
+ name: can_write
+ head_uuid: zzzzz-j7d0g-fffffffffffffff
+ properties: {}
ensure_unique_name: true
}
assert_response :success
- assert_match /^owned_by_active \(\d{4}-\d\d-\d\d.*?Z\)$/, json_response['name']
+ assert_match /^owned_by_active \(#{json_response['uuid'][-15..-1]}\)$/, json_response['name']
end
end
assert_equal false, json_response['is_trashed']
assert_nil json_response['trash_at']
assert_nil json_response['delete_at']
- assert_match /^same name for trashed and persisted collections \(\d{4}-\d\d-\d\d.*?Z\)$/, json_response['name']
+ assert_match /^same name for trashed and persisted collections \(#{json_response['uuid'][-15..-1]}\)$/, json_response['name']
end
test 'cannot show collection in trashed subproject' do
assert_not_equal(new_project['uuid'],
groups(:aproject).uuid,
"create returned same uuid as existing project")
- assert_match(/^A Project \(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3}Z\)$/,
+ assert_match(/^A Project \(#{new_project['uuid'][-15..-1]}\)$/,
new_project['name'])
end
ensure_unique_name: true
}
assert_response :success
- assert_match /^trashed subproject 3 \(\d{4}-\d\d-\d\d.*?Z\)$/, json_response['name']
+ assert_match /^trashed subproject 3 \(#{json_response['uuid'][-15..-1]}\)$/, json_response['name']
end
test "move trashed subproject to new owner #{auth}" do
'is_active' => true,
'is_admin' => true,
'prefs' => {'foo' => 'bar'},
+ 'is_invited' => true
},
newuuid => {
'first_name' => 'noot',
'email' => 'root@remot.example.com',
'username' => '',
+ 'is_invited' => true
},
unchanginguuid => {
'email' => 'root@unchanging.example.com',
'prefs' => {'foo' => {'bar' => 'baz'}},
+ 'is_invited' => true
},
conflictinguuid1 => {
'email' => 'root@conflictingname1.example.com',
- 'username' => 'active'
+ 'username' => 'active',
+ 'is_invited' => true
},
conflictinguuid2 => {
'email' => 'root@conflictingname2.example.com',
- 'username' => 'federatedactive'
+ 'username' => 'federatedactive',
+ 'is_invited' => true
},
}})
assert_response(:success)
assert_equal(1, Log.where(object_uuid: unchanginguuid).count)
end
- NON_ADMIN_USER_DATA = ["uuid", "kind", "is_active", "email", "first_name",
+ NON_ADMIN_USER_DATA = ["uuid", "kind", "is_active", "is_admin", "is_invited", "email", "first_name",
"last_name", "username", "can_write", "can_manage"].sort
def check_non_admin_index
assert_response :success
rp = json_response
assert_not_nil rp["uuid"]
- assert_not_nil rp["is_active"]
- assert_nil rp["is_admin"]
+ assert_equal true, rp["is_active"]
+ assert_equal false, rp["is_admin"]
get "/arvados/v1/users/#{rp['uuid']}",
params: {format: 'json'},
headers: auth(:admin)
assert_response :success
assert_equal rp["uuid"], json_response['uuid']
- assert_nil json_response['is_admin']
+ assert_equal false, json_response['is_admin']
assert_equal true, json_response['is_active']
assert_equal 'foo@example.com', json_response['email']
assert_equal 'barney', json_response['username']
assert_equal ContainerRequest::Final, cr.state
output_coll = Collection.find_by_uuid(cr.output_uuid)
# Make sure the resulting output collection name include the original name
- # plus the date
+ # plus the last 15 characters of uuid
assert_not_equal output_name, output_coll.name,
"more than one collection with the same owner and name"
assert output_coll.name.include?(output_name),
"New name should include original name"
- assert_match /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z/, output_coll.name,
- "New name should include ISO8601 date"
+ assert_match /#{output_coll.uuid[-15..-1]}/, output_coll.name,
+ "New name should include last 15 characters of uuid"
end
[[0, :check_output_ttl_0],
assert_equal("active/foo", repositories(:foo).name)
end
- [[false, 'foo@example.com', true, nil],
- [false, 'bar@example.com', nil, true],
- [true, 'foo@example.com', true, nil],
+ [[false, 'foo@example.com', true, false],
+ [false, 'bar@example.com', false, true],
+ [true, 'foo@example.com', true, false],
[true, 'bar@example.com', true, true],
- [false, '', nil, nil],
- [true, '', true, nil]
+ [false, '', false, false],
+ [true, '', true, false]
].each do |auto_admin_first_user_config, auto_admin_user_config, foo_should_be_admin, bar_should_be_admin|
# In each case, 'foo' is created first, then 'bar', then 'bar2', then 'baz'.
test "auto admin with auto_admin_first=#{auto_admin_first_user_config} auto_admin=#{auto_admin_user_config}" do
ARVADOS_ROOT="$ARVBOX_DATA/arvados"
fi
-if test -z "$WORKBENCH2_ROOT" ; then
- WORKBENCH2_ROOT="$ARVBOX_DATA/workbench2"
-fi
-
if test -z "$ARVADOS_BRANCH" ; then
ARVADOS_BRANCH=main
fi
-if test -z "$WORKBENCH2_BRANCH" ; then
- WORKBENCH2_BRANCH=main
-fi
-
# Update this to the docker tag for the version on releases.
DEFAULT_TAG=
docker_run_dev() {
docker run \
"--volume=$ARVADOS_ROOT:/usr/src/arvados:rw" \
- "--volume=$WORKBENCH2_ROOT:/usr/src/workbench2:rw" \
"--volume=$PG_DATA:/var/lib/postgresql:rw" \
"--volume=$VAR_DATA:$ARVADOS_CONTAINER_PATH:rw" \
"--volume=$PASSENGER:/var/lib/passenger:rw" \
git clone https://git.arvados.org/arvados.git "$ARVADOS_ROOT"
git -C "$ARVADOS_ROOT" checkout $ARVADOS_BRANCH
fi
- if ! test -d "$WORKBENCH2_ROOT" ; then
- git clone https://git.arvados.org/arvados-workbench2.git "$WORKBENCH2_ROOT"
- git -C "$ARVADOS_ROOT" checkout $WORKBENCH2_BRANCH
- fi
if [[ "$CONFIG" = test ]] ; then
docker build --build-arg=BUILDTYPE=$BUILDTYPE $NO_CACHE \
--build-arg=go_version=$GO_VERSION \
--build-arg=arvados_version=$ARVADOS_BRANCH \
- --build-arg=workbench2_version=$WORKBENCH2_BRANCH \
--build-arg=workdir=/tools/arvbox/lib/arvbox/docker \
-t arvados/arvbox-base:$GITHEAD \
-f "$ARVBOX_DOCKER/Dockerfile.base" \
docker build $NO_CACHE \
--build-arg=go_version=$GO_VERSION \
--build-arg=arvados_version=$ARVADOS_BRANCH \
- --build-arg=workbench2_version=$WORKBENCH2_BRANCH \
-t arvados/arvbox-$BUILDTYPE:$GITHEAD \
-f "$ARVBOX_DOCKER/Dockerfile.$BUILDTYPE" \
"$ARVBOX_DOCKER"
"$ARVBOX_BASE/$1/gopath" \
"$ARVBOX_BASE/$1/Rlibs" \
"$ARVBOX_BASE/$1/arvados" \
- "$ARVBOX_BASE/$1/workbench2" \
"$ARVBOX_BASE/$2"
echo "Created new arvbox $2"
echo "export ARVBOX_CONTAINER=$2"
FROM arvados/arvbox-base
ARG arvados_version
-ARG workbench2_version=main
RUN cd /usr/src && \
git clone --no-checkout https://git.arvados.org/arvados.git && \
git -C arvados checkout ${arvados_version} && \
- git clone --no-checkout https://git.arvados.org/arvados-workbench2.git workbench2 && \
- git -C workbench2 checkout ${workbench2_version} && \
chown -R 1000:1000 /usr/src
# avoid rebuilding arvados-server, it's already been built as part of the base image
done
fi
-cd /usr/src/workbench2
+cd /usr/src/arvados/services/workbench2
yarn install
API_HOST=${localip}:${services[controller-ssl]}
-if test -f /usr/src/workbench2/public/API_HOST ; then
- API_HOST=$(cat /usr/src/workbench2/public/API_HOST)
+if test -f /usr/src/arvados/services/workbench2/public/API_HOST ; then
+ API_HOST=$(cat /usr/src/arvados/services/workbench2/public/API_HOST)
fi
-cat <<EOF > /usr/src/workbench2/public/config.json
+cat <<EOF > /usr/src/arvados/services/workbench2/public/config.json
{
"API_HOST": "$API_HOST"
}
# Can't use "yarn start", need to run the dev server script
# directly so that the TERM signal from "sv restart" gets to the
# right process.
-export VERSION=$(./version-at-commit.sh)
+export VERSION=$(./version-at-commit.sh HEAD)
export BROWSER=none
export CI=true
export HTTPS=false
projects / arvados.git / commitdiff