X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/19f96717d0a7f26c28f8e5c61417c4246cfcffe1..f84e4b2ab7cd923aff2f99c04cb1313b36866393:/lib/config/config.default.yml diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml index d7a67e7721..a9bbf4eee9 100644 --- a/lib/config/config.default.yml +++ b/lib/config/config.default.yml @@ -24,49 +24,45 @@ Clusters: # In each of the service sections below, the keys under # InternalURLs are the endpoints where the service should be - # listening, and reachable from other hosts in the cluster. - SAMPLE: - InternalURLs: - "http://host1.example:12345": {} - "http://host2.example:12345": - # Rendezvous is normally empty/omitted. When changing the - # URL of a Keepstore service, Rendezvous should be set to - # the old URL (with trailing slash omitted) to preserve - # rendezvous ordering. - Rendezvous: "" - SAMPLE: - Rendezvous: "" - ExternalURL: "-" + # listening, and reachable from other hosts in the + # cluster. Example: + # + # InternalURLs: + # "http://host1.example:12345": {} + # "http://host2.example:12345": {} RailsAPI: - InternalURLs: {} - ExternalURL: "-" + InternalURLs: {SAMPLE: {}} + ExternalURL: "" Controller: - InternalURLs: {} + InternalURLs: {SAMPLE: {}} ExternalURL: "" Websocket: - InternalURLs: {} + InternalURLs: {SAMPLE: {}} ExternalURL: "" Keepbalance: - InternalURLs: {} - ExternalURL: "-" + InternalURLs: {SAMPLE: {}} + ExternalURL: "" GitHTTP: - InternalURLs: {} + InternalURLs: {SAMPLE: {}} ExternalURL: "" GitSSH: - InternalURLs: {} + InternalURLs: {SAMPLE: {}} ExternalURL: "" DispatchCloud: - InternalURLs: {} - ExternalURL: "-" - SSO: - InternalURLs: {} + InternalURLs: {SAMPLE: {}} + ExternalURL: "" + DispatchLSF: + InternalURLs: {SAMPLE: {}} + ExternalURL: "" + DispatchSLURM: + InternalURLs: {SAMPLE: {}} ExternalURL: "" Keepproxy: - InternalURLs: {} + InternalURLs: {SAMPLE: {}} ExternalURL: "" WebDAV: - InternalURLs: {} + InternalURLs: {SAMPLE: {}} # Base URL for Workbench inline preview. If blank, use # WebDAVDownload instead, and disable inline preview. # If both are empty, downloading collections from workbench @@ -105,7 +101,7 @@ Clusters: ExternalURL: "" WebDAVDownload: - InternalURLs: {} + InternalURLs: {SAMPLE: {}} # Base URL for download links. If blank, serve links to WebDAV # with disposition=attachment query param. Unlike preview links, # browsers do not render attachments, so there is no risk of XSS. @@ -119,13 +115,19 @@ Clusters: ExternalURL: "" Keepstore: - InternalURLs: {} - ExternalURL: "-" + InternalURLs: + SAMPLE: + # Rendezvous is normally empty/omitted. When changing the + # URL of a Keepstore service, Rendezvous should be set to + # the old URL (with trailing slash omitted) to preserve + # rendezvous ordering. + Rendezvous: "" + ExternalURL: "" Composer: - InternalURLs: {} + InternalURLs: {SAMPLE: {}} ExternalURL: "" WebShell: - InternalURLs: {} + InternalURLs: {SAMPLE: {}} # ShellInABox service endpoint URL for a given VM. If empty, do not # offer web shell logins. # @@ -136,14 +138,14 @@ Clusters: # https://*.webshell.uuid_prefix.arvadosapi.com ExternalURL: "" Workbench1: - InternalURLs: {} + InternalURLs: {SAMPLE: {}} ExternalURL: "" Workbench2: - InternalURLs: {} + InternalURLs: {SAMPLE: {}} ExternalURL: "" Health: - InternalURLs: {} - ExternalURL: "-" + InternalURLs: {SAMPLE: {}} + ExternalURL: "" PostgreSQL: # max concurrent connections per arvados server daemon @@ -235,6 +237,25 @@ Clusters: # Timeout on requests to internal Keep services. KeepServiceRequestTimeout: 15s + # Vocabulary file path, local to the node running the controller. + # This JSON file should contain the description of what's allowed + # as object's metadata. Its format is described at: + # https://doc.arvados.org/admin/metadata-vocabulary.html + VocabularyPath: "" + + # If true, a project must have a non-empty description field in + # order to be frozen. + FreezeProjectRequiresDescription: false + + # Project properties that must have non-empty values in order to + # freeze a project. Example: "property_name": {} + FreezeProjectRequiresProperties: + SAMPLE: {} + + # If true, only an admin user can un-freeze a project. If false, + # any user with "manage" permission can un-freeze. + UnfreezeProjectRequiresAdmin: false + Users: # Config parameters to automatically setup new users. If enabled, # this users will be able to self-activate. Enable this if you want @@ -260,6 +281,16 @@ Clusters: # user agreements. Should only be enabled for development. NewUsersAreActive: false + # Newly activated users (whether set up by an admin or via + # AutoSetupNewUsers) immediately become visible to other active + # users. + # + # On a multi-tenant cluster, where the intent is for users to be + # invisible to one another unless they have been added to the + # same group(s) via Workbench admin interface, change this to + # false. + ActivatedUsersAreVisibleToOthers: true + # The e-mail address of the user you would like to become marked as an admin # user on their first login. AutoAdminUserWithEmail: "" @@ -274,13 +305,12 @@ Clusters: AdminNotifierEmailFrom: arvados@example.com EmailSubjectPrefix: "[ARVADOS] " UserNotifierEmailFrom: arvados@example.com + UserNotifierEmailBcc: {} NewUserNotificationRecipients: {} NewInactiveUserNotificationRecipients: {} # Set AnonymousUserToken to enable anonymous user access. Populate this - # field with a long random string. Then run "bundle exec - # ./script/get_anonymous_user_token.rb" in the directory where your API - # server is running to record the token in the database. + # field with a random string at least 50 characters long. AnonymousUserToken: "" # If a new user has an alternate email address (local@domain) @@ -303,6 +333,14 @@ Clusters: Thanks, Your Arvados administrator. + # If RoleGroupsVisibleToAll is true, all role groups are visible + # to all active users. + # + # If false, users must be granted permission to role groups in + # order to see them. This is more appropriate for a multi-tenant + # cluster. + RoleGroupsVisibleToAll: true + AuditLogs: # Time to keep audit logs, in seconds. (An audit log is a row added # to the "logs" table in the PostgreSQL database each time an @@ -433,7 +471,7 @@ Clusters: # # BalancePeriod determines the interval between start times of # successive scan/balance operations. If a scan/balance operation - # takes longer than RunPeriod, the next one will follow it + # takes longer than BalancePeriod, the next one will follow it # immediately. # # If SIGUSR1 is received during an idle period between operations, @@ -460,6 +498,13 @@ Clusters: # long-running balancing operation. BalanceTimeout: 6h + # Maximum number of replication_confirmed / + # storage_classes_confirmed updates to write to the database + # after a rebalancing run. When many updates are needed, this + # spreads them over a few runs rather than applying them all at + # once. + BalanceUpdateLimit: 100000 + # Default lifetime for ephemeral collections: 2 weeks. This must not # be less than BlobSigningTTL. DefaultTrashLifetime: 336h @@ -474,12 +519,12 @@ Clusters: # is older than the amount of seconds defined on PreserveVersionIfIdle, # a snapshot of the collection's previous state is created and linked to # the current collection. - CollectionVersioning: false + CollectionVersioning: true # 0s = auto-create a new version on every update. # -1s = never auto-create new versions. # > 0s = auto-create a new version when older than the specified number of seconds. - PreserveVersionIfIdle: -1s + PreserveVersionIfIdle: 10s # If non-empty, allow project and collection names to contain # the "/" character (slash/stroke/solidus), and replace "/" with @@ -521,10 +566,10 @@ Clusters: # WebDAV would have to expose XSS vulnerabilities in order to # handle the redirect (see discussion on Services.WebDAV). # - # This setting has no effect in the recommended configuration, - # where the WebDAV is configured to have a separate domain for - # every collection; in this case XSS protection is provided by - # browsers' same-origin policy. + # This setting has no effect in the recommended configuration, where the + # WebDAV service is configured to have a separate domain for every + # collection and XSS protection is provided by browsers' same-origin + # policy. # # The default setting (false) is appropriate for a multi-user site. TrustAllContent: false @@ -538,7 +583,7 @@ Clusters: UUIDTTL: 5s # Block cache entries. Each block consumes up to 64 MiB RAM. - MaxBlockEntries: 4 + MaxBlockEntries: 20 # Collection cache entries. MaxCollectionEntries: 1000 @@ -546,17 +591,42 @@ Clusters: # Approximate memory limit (in bytes) for collection cache. MaxCollectionBytes: 100000000 - # Permission cache entries. - MaxPermissionEntries: 1000 - # UUID cache entries. MaxUUIDEntries: 1000 # Persistent sessions. MaxSessions: 100 + # Selectively set permissions for regular users and admins to + # download or upload data files using the upload/download + # features for Workbench, WebDAV and S3 API support. + WebDAVPermission: + User: + Download: true + Upload: true + Admin: + Download: true + Upload: true + + # Selectively set permissions for regular users and admins to be + # able to download or upload blocks using arv-put and + # arv-get from outside the cluster. + KeepproxyPermission: + User: + Download: true + Upload: true + Admin: + Download: true + Upload: true + + # Post upload / download events to the API server logs table, so + # that they can be included in the arv-user-activity report. + # You can disable this if you find that it is creating excess + # load on the API server and you don't need it. + WebDAVLogEvents: true + Login: - # One of the following mechanisms (SSO, Google, PAM, LDAP, or + # One of the following mechanisms (Google, PAM, LDAP, or # LoginCluster) should be enabled; see # https://doc.arvados.org/install/setup-login.html @@ -633,8 +703,25 @@ Clusters: AuthenticationRequestParameters: SAMPLE: "" + # Accept an OIDC access token as an API token if the OIDC + # provider's UserInfo endpoint accepts it. + # + # AcceptAccessTokenScope should also be used when enabling + # this feature. + AcceptAccessToken: false + + # Before accepting an OIDC access token as an API token, first + # check that it is a JWT whose "scope" value includes this + # value. Example: "https://zzzzz.example.com/" (your Arvados + # API endpoint). + # + # If this value is empty and AcceptAccessToken is true, all + # access tokens will be accepted regardless of scope, + # including non-JWT tokens. This is not recommended. + AcceptAccessTokenScope: "" + PAM: - # (Experimental) Use PAM to authenticate users. + # Use PAM to authenticate users. Enable: false # PAM service name. PAM will apply the policy in the @@ -720,16 +807,6 @@ Clusters: # originally supplied by the user will be used. UsernameAttribute: uid - SSO: - # Authenticate with a separate SSO server. (Deprecated) - Enable: false - - # ProviderAppID and ProviderAppSecret are generated during SSO - # setup; see - # https://doc.arvados.org/v2.0/install/install-sso.html#update-config - ProviderAppID: "" - ProviderAppSecret: "" - Test: # Authenticate users listed here in the config file. This # feature is intended to be used in test environments, and @@ -754,8 +831,15 @@ Clusters: # Default value zero means tokens don't have expiration. TokenLifetime: 0s + # If true (default) tokens issued through login are allowed to create + # new tokens. + # If false, tokens issued through login are not allowed to + # viewing/creating other tokens. New tokens can only be created + # by going through login again. + IssueTrustedTokens: true + # When the token is returned to a client, the token itself may - # be restricted from manipulating other tokens based on whether + # be restricted from viewing/creating other tokens based on whether # the client is "trusted" or not. The local Workbench1 and # Workbench2 are trusted by default, but if this is a # LoginCluster, you probably want to include the other Workbench @@ -829,14 +913,28 @@ Clusters: # go down. MaxComputeVMs: 64 - # Preemptible instance support (e.g. AWS Spot Instances) - # When true, child containers will get created with the preemptible - # scheduling parameter parameter set. - UsePreemptibleInstances: false + # Schedule all child containers on preemptible instances (e.g. AWS + # Spot Instances) even if not requested by the submitter. + # + # If false, containers are scheduled on preemptible instances + # only when requested by the submitter. + # + # This flag is ignored if no preemptible instance types are + # configured, and has no effect on top-level containers. + AlwaysUsePreemptibleInstances: false + + # Automatically add a preemptible variant for every + # non-preemptible entry in InstanceTypes below. The maximum bid + # price for the preemptible variant will be the non-preemptible + # price multiplied by PreemptiblePriceFactor. If 0, preemptible + # variants are not added automatically. + # + # A price factor of 1.0 is a reasonable starting point. + PreemptiblePriceFactor: 0 # PEM encoded SSH key (RSA, DSA, or ECDSA) used by the - # (experimental) cloud dispatcher for executing containers on - # worker VMs. Begins with "-----BEGIN RSA PRIVATE KEY-----\n" + # cloud dispatcher for executing containers on worker VMs. + # Begins with "-----BEGIN RSA PRIVATE KEY-----\n" # and ends with "\n-----END RSA PRIVATE KEY-----\n". DispatchPrivateKey: "" @@ -862,6 +960,55 @@ Clusters: # Minimum time between two attempts to run the same container MinRetryPeriod: 0s + # Container runtime: "docker" (default) or "singularity" + RuntimeEngine: docker + + # When running a container, run a dedicated keepstore process, + # using the specified number of 64 MiB memory buffers per + # allocated CPU core (VCPUs in the container's runtime + # constraints). The dedicated keepstore handles I/O for + # collections mounted in the container, as well as saving + # container logs. + # + # A zero value disables this feature. + # + # In order for this feature to be activated, no volume may use + # AccessViaHosts, and no writable volume may have Replication + # lower than Collections.DefaultReplication. If these + # requirements are not satisfied, the feature is disabled + # automatically regardless of the value given here. + # + # When an HPC dispatcher is in use (see SLURM and LSF sections), + # this feature depends on the operator to ensure an up-to-date + # cluster configuration file (/etc/arvados/config.yml) is + # available on all compute nodes. If it is missing or not + # readable by the crunch-run user, the feature will be disabled + # automatically. To read it from a different location, add a + # "-config=/path/to/config.yml" argument to + # CrunchRunArgumentsList above. + # + # When the cloud dispatcher is in use (see CloudVMs section) and + # this configuration is enabled, the entire cluster + # configuration file, including the system root token, is copied + # to the worker node and held in memory for the duration of the + # container. + LocalKeepBlobBuffersPerVCPU: 1 + + # When running a dedicated keepstore process for a container + # (see LocalKeepBlobBuffersPerVCPU), write keepstore log + # messages to keepstore.txt in the container's log collection. + # + # These log messages can reveal some volume configuration + # details, error messages from the cloud storage provider, etc., + # which are not otherwise visible to users. + # + # Accepted values: + # * "none" -- no keepstore.txt file + # * "all" -- all logs, including request and response lines + # * "errors" -- all logs except "response" logs with 2xx + # response codes and "request" logs + LocalKeepLogsToContainerLog: none + Logging: # When you run the db:delete_old_container_logs task, it will find # containers that have been finished for at least this many seconds, @@ -971,6 +1118,39 @@ Clusters: # (See http://ruby-doc.org/core-2.2.2/Kernel.html#method-i-format for more.) AssignNodeHostname: "compute%d" + LSF: + # Arguments to bsub when submitting Arvados containers as LSF jobs. + # + # Template variables starting with % will be substituted as follows: + # + # %U uuid + # %C number of VCPUs + # %M memory in MB + # %T tmp in MB + # %G number of GPU devices (runtime_constraints.cuda.device_count) + # + # Use %% to express a literal %. The %%J in the default will be changed + # to %J, which is interpreted by bsub itself. + # + # Note that the default arguments cause LSF to write two files + # in /tmp on the compute node each time an Arvados container + # runs. Ensure you have something in place to delete old files + # from /tmp, or adjust the "-o" and "-e" arguments accordingly. + BsubArgumentsList: ["-o", "/tmp/crunch-run.%%J.out", "-e", "/tmp/crunch-run.%%J.err", "-J", "%U", "-n", "%C", "-D", "%MMB", "-R", "rusage[mem=%MMB:tmp=%TMB] span[hosts=1]", "-R", "select[mem>=%MMB]", "-R", "select[tmp>=%TMB]", "-R", "select[ncpus>=%C]"] + + # Arguments that will be appended to the bsub command line + # when submitting Arvados containers as LSF jobs with + # runtime_constraints.cuda.device_count > 0 + BsubCUDAArguments: ["-gpu", "num=%G"] + + # Use sudo to switch to this user account when submitting LSF + # jobs. + # + # This account must exist on the hosts where LSF jobs run + # ("execution hosts"), as well as on the host where the + # Arvados LSF dispatcher runs ("submission host"). + BsubSudoUser: "crunch" + JobsAPI: # Enable the legacy 'jobs' API (crunch v1). This value must be a string. # @@ -990,7 +1170,7 @@ Clusters: GitInternalDir: /var/lib/arvados/internal.git CloudVMs: - # Enable the cloud scheduler (experimental). + # Enable the cloud scheduler. Enable: false # Name/number of port where workers' SSH services listen. @@ -1002,7 +1182,7 @@ Clusters: # Shell command to execute on each worker to determine whether # the worker is booted and ready to run containers. It should # exit zero if the worker is ready. - BootProbeCommand: "docker ps -q" + BootProbeCommand: "systemctl is-system-running" # Minimum interval between consecutive probes to a single # worker. @@ -1024,13 +1204,25 @@ Clusters: # Maximum create/destroy-instance operations per second (0 = # unlimited). - MaxCloudOpsPerSecond: 0 + MaxCloudOpsPerSecond: 10 - # Maximum concurrent node creation operations (0 = unlimited). This is - # recommended by Azure in certain scenarios (see - # https://docs.microsoft.com/en-us/azure/virtual-machines/linux/capture-image) - # and can be used with other cloud providers too, if desired. - MaxConcurrentInstanceCreateOps: 0 + # Maximum concurrent instance creation operations (0 = unlimited). + # + # MaxConcurrentInstanceCreateOps limits the number of instance creation + # requests that can be in flight at any one time, whereas + # MaxCloudOpsPerSecond limits the number of create/destroy operations + # that can be started per second. + # + # Because the API for instance creation on Azure is synchronous, it is + # recommended to increase MaxConcurrentInstanceCreateOps when running + # on Azure. When using managed images, a value of 20 would be + # appropriate. When using Azure Shared Image Galeries, it could be set + # higher. For more information, see + # https://docs.microsoft.com/en-us/azure/virtual-machines/linux/capture-image + # + # MaxConcurrentInstanceCreateOps can be increased for other cloud + # providers too, if desired. + MaxConcurrentInstanceCreateOps: 1 # Interval between cloud provider syncs/updates ("list all # instances"). @@ -1090,7 +1282,9 @@ Clusters: # need to be detected and cleaned up manually. TagKeyPrefix: Arvados - # Cloud driver: "azure" (Microsoft Azure) or "ec2" (Amazon AWS). + # Cloud driver: "azure" (Microsoft Azure), "ec2" (Amazon AWS), + # or "loopback" (run containers on dispatch host for testing + # purposes). Driver: ec2 # Cloud-specific driver parameters. @@ -1107,6 +1301,9 @@ Clusters: Region: "" EBSVolumeType: gp2 AdminUsername: debian + # (ec2) name of the IAMInstanceProfile for instances started by + # the cloud dispatcher. Leave blank when not needed. + IAMInstanceProfile: "" # (azure) Credentials. SubscriptionID: "" @@ -1163,6 +1360,34 @@ Clusters: AddedScratch: 0 Price: 0.1 Preemptible: false + # Include this section if the node type includes GPU (CUDA) support + CUDA: + DriverVersion: "11.0" + HardwareCapability: "9.0" + DeviceCount: 1 + + StorageClasses: + + # If you use multiple storage classes, specify them here, using + # the storage class name as the key (in place of "SAMPLE" in + # this sample entry). + # + # Further info/examples: + # https://doc.arvados.org/admin/storage-classes.html + SAMPLE: + + # Priority determines the order volumes should be searched + # when reading data, in cases where a keepstore server has + # access to multiple volumes with different storage classes. + Priority: 0 + + # Default determines which storage class(es) should be used + # when a user/client writes data or saves a new collection + # without specifying storage classes. + # + # If any StorageClasses are configured, at least one of them + # must have Default: true. + Default: true Volumes: SAMPLE: @@ -1187,7 +1412,9 @@ Clusters: ReadOnly: false Replication: 1 StorageClasses: - default: true + # If you have configured storage classes (see StorageClasses + # section above), add an entry here for each storage class + # satisfied by this volume. SAMPLE: true Driver: S3 DriverParameters: @@ -1197,7 +1424,7 @@ Clusters: AccessKeyID: aaaaa SecretAccessKey: aaaaa Endpoint: "" - Region: us-east-1a + Region: us-east-1 Bucket: aaaaa LocationConstraint: false V2Signature: false @@ -1205,6 +1432,7 @@ Clusters: ConnectTimeout: 1m ReadTimeout: 10m RaceWindow: 24h + PrefixLength: 0 # Use aws-s3-go (v2) instead of goamz UseAWSS3v2Driver: false @@ -1298,6 +1526,11 @@ Clusters: ShowUserAgreementInline: false SecretKeyBase: "" + # Set this configuration to true to avoid providing an easy way for users + # to share data with unauthenticated users; this may be necessary on + # installations where strict data access controls are needed. + DisableSharingURLsUI: false + # Scratch directory used by the remote repository browsing # feature. If it doesn't exist, it (and any missing parents) will be # created using mkdir_p. @@ -1416,7 +1649,6 @@ Clusters: DefaultOpenIdPrefix: "https://www.google.com/accounts/o8/id" # Workbench2 configs - VocabularyURL: "" FileViewersConfigURL: "" # Idle time after which the user's session will be auto closed. @@ -1429,15 +1661,11 @@ Clusters:

Please log in.

-

The "Log in" button below will show you a sign-in - page. After you log in, you will be redirected back to - Arvados Workbench.

-

If you have never used Arvados Workbench before, logging in for the first time will automatically create a new account.

- Arvados Workbench uses your name and email address only for + Arvados Workbench uses your information only for identification, and does not retrieve any other personal information.