- api/methods/containers.html.textile.liquid
- api/methods/workflows.html.textile.liquid
- api/dispatch.html.textile.liquid
- - Jobs engine (legacy):
- - api/crunch-scripts.html.textile.liquid
- - api/methods/jobs.html.textile.liquid
- - api/methods/job_tasks.html.textile.liquid
- - api/methods/pipeline_instances.html.textile.liquid
- - api/methods/pipeline_templates.html.textile.liquid
- - api/methods/nodes.html.textile.liquid
- - api/methods/keep_disks.html.textile.liquid
- - Metadata for bioinformatics (legacy):
- - api/methods/humans.html.textile.liquid
- - api/methods/specimens.html.textile.liquid
- - api/methods/traits.html.textile.liquid
architecture:
- Topics:
- architecture/index.html.textile.liquid
import arvados.util as util
filters = [['properties.responsible_person_uuid', 'exists', False]]
-cols = util.list_all(arvados.api().collections().list, filters=filters, select=['uuid', 'name'])
+cols = util.keyset_list_all(arvados.api().collections().list, filters=filters, select=['uuid', 'name'], order='uuid')
print('Found {} collections:'.format(len(cols)))
for c in cols:
- print('{}, "{}"'.format(c['uuid'], c['name']))
\ No newline at end of file
+ print('{}, "{}"'.format(c['uuid'], c['name']))
def get_subproject_uuids(api, root_uuid):
uuids = []
- groups = util.list_all(api.groups().list, filters=[['owner_uuid', '=', '{}'.format(root_uuid)]], select=['uuid'])
+ groups = util.keyset_list_all(api.groups().list, filters=[['owner_uuid', '=', '{}'.format(root_uuid)]], select=['uuid'], order='uuid')
for g in groups:
uuids += ([g['uuid']] + get_subproject_uuids(api, g['uuid']))
return uuids
def get_cols(api, filters):
- cols = util.list_all(api.collections().list, filters=filters, select=['uuid', 'properties'])
+ cols = util.keyset_list_all(api.collections().list, filters=filters, select=['uuid', 'properties'], order='uuid')
return cols
# Search for collections on project hierarchy rooted at root_uuid
print(' - Updating collection {}'.format(c['uuid']))
props = c['properties']
props['responsible_person_uuid'] = responsible_uuid
- api.collections().update(uuid=c['uuid'], body={'properties': props}).execute()
\ No newline at end of file
+ api.collections().update(uuid=c['uuid'], body={'properties': props}).execute()
api = arvados.api()
filters = [['properties.responsible_person_uuid', '=', '{}'.format(old_uuid)]]
-cols = util.list_all(api.collections().list, filters=filters, select=['uuid', 'properties'])
+cols = util.keyset_list_all(api.collections().list, filters=filters, select=['uuid', 'properties'], order='uuid')
print('Found {} collections'.format(len(cols)))
for c in cols:
print('Updating collection {}'.format(c['uuid']))
props = c['properties']
props['responsible_person_uuid'] = new_uuid
- api.collections().update(uuid=c['uuid'], body={'properties': props}).execute()
\ No newline at end of file
+ api.collections().update(uuid=c['uuid'], body={'properties': props}).execute()
"modified_by_client_uuid":null,
"modified_by_user_uuid":null,
"modified_at":null,
- "user_id":3,
"api_client_id":7,
"api_token":"5a74htnoqwkhtfo2upekpfbsg04hv7cy5v4nowf7dtpxer086m",
"created_by_ip_address":null,
- "default_owner_uuid":null,
"expires_at":null,
"last_used_at":null,
"last_used_by_ip_address":null,
If you have a custom install that requires a different version of Ruby than the one included with your distribution, you must configure your system to ensure package scripts find that version of @ruby@ before any others. For example, you might do this on Debian-based distributions by customizing apt's @DPkg::Path@ setting.
+h3. Configuration entries have been removed or renamed
+
+The following configuration keys have been renamed or removed. Renamed keys will still be loaded if they appear with their old names, but you should update your @/etc/arvados/config.yml@ file to avoid warnings when services start up.
+* @Containers.JobsAPI.Enable@ has been removed
+* @Mail.EmailFrom@ has been removed
+* @Mail.IssueReporterEmailFrom@ has been removed
+* @Mail.IssueReporterEmailTo@ has been removed
+* @Mail.MailchimpAPIKey@ has been removed
+* @Mail.MailchimpListID@ has been removed
+* @Mail.SendUserSetupNotificationEmail@ has moved to @Users.SendUserSetupNotificationEmail@
+* @Mail.SupportEmailAddress@ has moved to @Users.SupportEmailAddress@
+
h3. S3 volume IAMRole configuration entry has been removed
The @Volumes.*.DriverParameters.IAMRole@ configuration entry for S3 volumes has been removed. You should remove it from your @/etc/arvados/config.yml@ file to avoid warnings when services start up. As before, if @AccessKeyID@ and @SecretAccessKey@ are blank, keepstore will retrieve IAM role credentials from instance metadata. Previously, documentation indicated that keepstore would refuse to use the IAM credentials if @IAMRole@ was specified and did not match the instance metadata, but that check has not been working for some time.
"modified_by_client_uuid":null,
"modified_by_user_uuid":null,
"modified_at":null,
- "user_id":3,
"api_client_id":7,
"api_token":"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
"created_by_ip_address":null,
- "default_owner_uuid":null,
"expires_at":null,
"last_used_at":null,
"last_used_by_ip_address":null,
|identity_url|string|||
|is_admin|boolean|||
|prefs|hash|||
-|default_owner_uuid|string|||
|is_active|boolean|||
|writable_by|array|List of UUID strings identifying Groups and other Users that can modify this User object. This will include the user's owner_uuid and, for administrators and users requesting their own User object, the requesting user's UUID.||
Before attempting installation, you should begin by reviewing supported platforms, choosing backends for identity, storage, and scheduling, and decide how you will distribute Arvados services onto machines. You should also choose an Arvados Cluster ID, choose your hostnames, and aquire TLS certificates. It may be helpful to make notes as you go along using one of these worksheets: "New cluster checklist for AWS":new_cluster_checklist_AWS.xlsx - "New cluster checklist for Azure":new_cluster_checklist_Azure.xlsx - "New cluster checklist for on premises Slurm":new_cluster_checklist_slurm.xlsx
-The installation guide describes how to set up a basic standalone Arvados instance. Additional configuration for features including "federation,":{{site.baseurl}}/admin/federation.html "collection versioning,":{{site.baseurl}}/admin/collection-versioning.html "managed properties,":{{site.baseurl}}/admin/collection-managed-properties.html and "storage classes":{{site.baseurl}}/admin/collection-managed-properties.html are described in the "Admin guide.":{{site.baseurl}}/admin
+The installation guide describes how to set up a basic standalone Arvados instance. Additional configuration for features including "federation,":{{site.baseurl}}/admin/federation.html "collection versioning,":{{site.baseurl}}/admin/collection-versioning.html "managed properties,":{{site.baseurl}}/admin/collection-managed-properties.html and "storage classes":{{site.baseurl}}/admin/collection-managed-properties.html are described in the "Admin guide.":{{site.baseurl}}/admin/
The Arvados storage subsystem is called "keep". The compute subsystem is called "crunch".
h2(#components). Choosing which components to install
-Arvados consists of many components, some of which may be omitted (at the cost of reduced functionality.) It may also be helpful to review the "Arvados Architecture":{{site.baseurl}}/architecture to understand how these components interact.
+Arvados consists of many components, some of which may be omitted (at the cost of reduced functionality.) It may also be helpful to review the "Arvados Architecture":{{site.baseurl}}/architecture/ to understand how these components interact.
table(table table-bordered table-condensed).
|\3=. *Core*|
id := flags.Args()[0]
client := arvados.NewClientFromEnv()
- path, err := client.PathForUUID("show", id)
+ path, err := client.PathForUUID("get", id)
if err != nil {
return 1
}
# other admin users exist will automatically become an admin user.
AutoAdminFirstUser: false
+ # Support email address to display in Workbench.
+ SupportEmailAddress: "arvados@example.com"
+
+ # Outgoing email configuration:
+ #
+ # In order to send mail, Arvados expects a default SMTP server
+ # on localhost:25. It cannot require authentication on
+ # connections from localhost. That server should be configured
+ # to relay mail to a "real" SMTP server that is able to send
+ # email on behalf of your domain.
+
# Recipient for notification email sent out when a user sets a
# profile on their account.
UserProfileNotificationAddress: ""
# Currently implemented for OpenID Connect only.
PreferDomainForUsername: ""
+ # Send an email to each user when their account has been set up
+ # (meaning they are able to log in).
+ SendUserSetupNotificationEmail: true
+
# Ruby ERB template used for the email sent out to users when
# they have been set up.
UserSetupMailText: |
# MaxRunTimeDefault: 2h
MaxRunTimeDefault: 0
- JobsAPI:
- # Enable the legacy 'jobs' API (crunch v1). This value must be a string.
- #
- # Note: this only enables read-only access, creating new
- # legacy jobs and pipelines is not supported.
- #
- # 'auto' -- (default) enable the Jobs API only if it has been used before
- # (i.e., there are job records in the database)
- # 'true' -- enable the Jobs API despite lack of existing records.
- # 'false' -- disable the Jobs API despite presence of existing records.
- Enable: 'auto'
-
CloudVMs:
# Enable the cloud scheduler.
Enable: false
# should leave this alone.
Serialize: false
- Mail:
- # In order to send mail, Arvados expects a default SMTP server
- # on localhost:25. It cannot require authentication on
- # connections from localhost. That server should be configured
- # to relay mail to a "real" SMTP server that is able to send
- # email on behalf of your domain.
-
- # See also the "Users" configuration section for additional
- # email-related options.
-
- # When a user has been set up (meaning they are able to log in)
- # they will receive an email using the template specified
- # earlier in Users.UserSetupMailText
- SendUserSetupNotificationEmail: true
-
- # Bug/issue report notification to and from addresses
- IssueReporterEmailFrom: "arvados@example.com"
- IssueReporterEmailTo: "arvados@example.com"
- SupportEmailAddress: "arvados@example.com"
-
- # Generic issue email from
- EmailFrom: "arvados@example.com"
-
- # No longer supported, to be removed.
- MailchimpAPIKey: ""
- MailchimpListID: ""
RemoteClusters:
"*":
Host: ""
ProviderAppID *string
ProviderAppSecret *string
}
+ Mail struct {
+ SendUserSetupNotificationEmail *bool
+ SupportEmailAddress *string
+ }
}
type deprecatedConfig struct {
if dst, n := &cluster.API.MaxRequestAmplification, dcluster.RequestLimits.MultiClusterRequestConcurrency; n != nil && *n != *dst {
*dst = *n
}
+ if dst, addr := &cluster.Users.SupportEmailAddress, dcluster.Mail.SupportEmailAddress; addr != nil {
+ *dst = *addr
+ ldr.Logger.Warnf("using your old config key Mail.SupportEmailAddress -- but you should rename it to Users.SupportEmailAddress")
+ }
+ if dst, b := &cluster.Users.SendUserSetupNotificationEmail, dcluster.Mail.SendUserSetupNotificationEmail; b != nil {
+ *dst = *b
+ ldr.Logger.Warnf("using your old config key Mail.SendUserSetupNotificationEmail -- but you should rename it to Users.SendUserSetupNotificationEmail")
+ }
// Google* moved to Google.*
if dst, n := &cluster.Login.Google.ClientID, dcluster.Login.GoogleClientID; n != nil && *n != *dst {
return cluster, nil
}
+func (s *LoadSuite) TestOldEmailConfiguration(c *check.C) {
+ logs := checkEquivalent(c, `
+Clusters:
+ z1111:
+ Mail:
+ SendUserSetupNotificationEmail: false
+ SupportEmailAddress: "support@example.invalid"
+`, `
+Clusters:
+ z1111:
+ Users:
+ SendUserSetupNotificationEmail: false
+ SupportEmailAddress: "support@example.invalid"
+`)
+ c.Check(logs, check.Matches, `(?ms).*deprecated or unknown config entry: .*Mail\.SendUserSetupNotificationEmail.*`)
+ c.Check(logs, check.Matches, `(?ms).*deprecated or unknown config entry: .*Mail\.SupportEmailAddress.*`)
+ c.Check(logs, check.Matches, `(?ms).*using your old config key Mail\.SendUserSetupNotificationEmail -- but you should rename it to Users\.SendUserSetupNotificationEmail.*`)
+ c.Check(logs, check.Matches, `(?ms).*using your old config key Mail\.SupportEmailAddress -- but you should rename it to Users\.SupportEmailAddress.*`)
+}
+
func (s *LoadSuite) TestLegacyVolumeDriverParameters(c *check.C) {
logs := checkEquivalent(c, `
Clusters:
"Containers.CrunchRunCommand": false,
"Containers.DefaultKeepCacheRAM": true,
"Containers.DispatchPrivateKey": false,
- "Containers.JobsAPI": true,
- "Containers.JobsAPI.Enable": true,
"Containers.LocalKeepBlobBuffersPerVCPU": false,
"Containers.LocalKeepLogsToContainerLog": false,
"Containers.Logging": false,
"Login.TokenLifetime": false,
"Login.TrustedClients": false,
"Login.TrustPrivateNetworks": false,
- "Mail": true,
- "Mail.EmailFrom": false,
- "Mail.IssueReporterEmailFrom": false,
- "Mail.IssueReporterEmailTo": false,
- "Mail.MailchimpAPIKey": false,
- "Mail.MailchimpListID": false,
- "Mail.SendUserSetupNotificationEmail": false,
- "Mail.SupportEmailAddress": true,
"ManagementToken": false,
"PostgreSQL": false,
"RemoteClusters": true,
"Users.NewUsersAreActive": false,
"Users.PreferDomainForUsername": false,
"Users.RoleGroupsVisibleToAll": false,
+ "Users.SendUserSetupNotificationEmail": false,
+ "Users.SupportEmailAddress": true,
"Users.SyncIgnoredGroups": true,
"Users.SyncRequiredGroups": true,
"Users.SyncUserAccounts": true,
"api_client_authorization": {
"owner_uuid": "`+arvadostest.AdminUserUUID+`",
"created_by_ip_address": "::1",
- "last_used_by_ip_address": "::1",
- "default_owner_uuid": "`+arvadostest.AdminUserUUID+`"
+ "last_used_by_ip_address": "::1"
}
}`))
req.Header.Set("Authorization", "Bearer "+arvadostest.SystemRootToken)
```r
?collections_update
-?jobs_get
+?workflows_get
```
<!-- Taka konwencja USAGE -->
stubs.fake_user_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
stubs.fake_container_uuid = "zzzzz-dz642-zzzzzzzzzzzzzzz"
- if sys.version_info[0] < 3:
- stubs.capture_stdout = BytesIO()
- else:
- stubs.capture_stdout = StringIO()
+ stubs.capture_stdout = StringIO()
stubs.api = mock.MagicMock()
stubs.api._rootDesc = get_rootDesc()
APIToken string `json:"api_token"`
CreatedAt time.Time `json:"created_at"`
CreatedByIPAddress string `json:"created_by_ip_address"`
- DefaultOwnerUUID string `json:"default_owner_uuid"`
Etag string `json:"etag"`
ExpiresAt time.Time `json:"expires_at"`
LastUsedAt time.Time `json:"last_used_at"`
ModifiedByUserUUID string `json:"modified_by_user_uuid"`
OwnerUUID string `json:"owner_uuid"`
Scopes []string `json:"scopes"`
- UserID int `json:"user_id"`
}
// APIClientAuthorizationList is an arvados#apiClientAuthorizationList resource.
TrustPrivateNetworks bool
IssueTrustedTokens bool
}
- Mail struct {
- MailchimpAPIKey string
- MailchimpListID string
- SendUserSetupNotificationEmail bool
- IssueReporterEmailFrom string
- IssueReporterEmailTo string
- SupportEmailAddress string
- EmailFrom string
- }
SystemLogs struct {
LogLevel string
Format string
NewInactiveUserNotificationRecipients StringSet
NewUserNotificationRecipients StringSet
NewUsersAreActive bool
+ SendUserSetupNotificationEmail bool
+ SupportEmailAddress string
UserNotifierEmailFrom string
UserNotifierEmailBcc StringSet
UserProfileNotificationAddress string
LocalKeepBlobBuffersPerVCPU int
LocalKeepLogsToContainerLog string
- JobsAPI struct {
- Enable string
- }
Logging struct {
LogUpdatePeriod Duration
LogUpdateSize ByteSize
+++ /dev/null
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: Apache-2.0
-
-package arvados
-
-import "time"
-
-// Job is an arvados#job record
-type Job struct {
- UUID string `json:"uuid"`
- Etag string `json:"etag"`
- OwnerUUID string `json:"owner_uuid"`
- ModifiedByClientUUID string `json:"modified_by_client_uuid"`
- ModifiedByUserUUID string `json:"modified_by_user_uuid"`
- ModifiedAt time.Time `json:"modified_at"`
- SubmitID string `json:"submit_id"`
- Script string `json:"script"`
- CancelledByClientUUID string `json:"cancelled_by_client_uuid"`
- CancelledByUserUUID string `json:"cancelled_by_user_uuid"`
- CancelledAt time.Time `json:"cancelled_at"`
- StartedAt time.Time `json:"started_at"`
- FinishedAt time.Time `json:"finished_at"`
- Running bool `json:"running"`
- Success bool `json:"success"`
- Output string `json:"output"`
- CreatedAt time.Time `json:"created_at"`
- UpdatedAt time.Time `json:"updated_at"`
- IsLockedByUUID string `json:"is_locked_by_uuid"`
- Log string `json:"log"`
- TasksSummary map[string]interface{} `json:"tasks_summary"`
- RuntimeConstraints map[string]interface{} `json:"runtime_constraints"`
- Nondeterministic bool `json:"nondeterministic"`
- Repository string `json:"repository"`
- SuppliedScriptVersion string `json:"supplied_script_version"`
- DockerImageLocator string `json:"docker_image_locator"`
- Priority int `json:"priority"`
- Description string `json:"description"`
- State string `json:"state"`
- ArvadosSDKVersion string `json:"arvados_sdk_version"`
- Components map[string]interface{} `json:"components"`
- ScriptParametersDigest string `json:"script_parameters_digest"`
- WritableBy []string `json:"writable_by,omitempty"`
-}
-
-func (g Job) resourceName() string {
- return "job"
-}
+++ /dev/null
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: Apache-2.0
-
-package arvados
-
-import "time"
-
-// PipelineInstance is an arvados#pipelineInstance record
-type PipelineInstance struct {
- UUID string `json:"uuid"`
- Etag string `json:"etag"`
- OwnerUUID string `json:"owner_uuid"`
- CreatedAt time.Time `json:"created_at"`
- ModifiedByClientUUID string `json:"modified_by_client_uuid"`
- ModifiedByUserUUID string `json:"modified_by_user_uuid"`
- ModifiedAt time.Time `json:"modified_at"`
- PipelineTemplateUUID string `json:"pipeline_template_uuid"`
- Name string `json:"name"`
- Components map[string]interface{} `json:"components"`
- UpdatedAt time.Time `json:"updated_at"`
- Properties map[string]interface{} `json:"properties"`
- State string `json:"state"`
- ComponentsSummary map[string]interface{} `json:"components_summary"`
- StartedAt time.Time `json:"started_at"`
- FinishedAt time.Time `json:"finished_at"`
- Description string `json:"description"`
- WritableBy []string `json:"writable_by,omitempty"`
-}
-
-func (g PipelineInstance) resourceName() string {
- return "pipelineInstance"
-}
+++ /dev/null
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: Apache-2.0
-
-package arvados
-
-import "time"
-
-// PipelineTemplate is an arvados#pipelineTemplate record
-type PipelineTemplate struct {
- UUID string `json:"uuid"`
- Etag string `json:"etag"`
- OwnerUUID string `json:"owner_uuid"`
- CreatedAt time.Time `json:"created_at"`
- ModifiedByClientUUID string `json:"modified_by_client_uuid"`
- ModifiedByUserUUID string `json:"modified_by_user_uuid"`
- ModifiedAt time.Time `json:"modified_at"`
- Name string `json:"name"`
- Components map[string]interface{} `json:"components"`
- UpdatedAt time.Time `json:"updated_at"`
- Description string `json:"description"`
- WritableBy []string `json:"writable_by,omitempty"`
-}
-
-func (g PipelineTemplate) resourceName() string {
- return "pipelineTemplate"
-}
+++ /dev/null
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: Apache-2.0
-
-package arvados
-
-import "time"
-
-// Trait is an arvados#trait record
-type Trait struct {
- UUID string `json:"uuid"`
- Etag string `json:"etag"`
- OwnerUUID string `json:"owner_uuid"`
- CreatedAt time.Time `json:"created_at"`
- ModifiedByClientUUID string `json:"modified_by_client_uuid"`
- ModifiedByUserUUID string `json:"modified_by_user_uuid"`
- ModifiedAt time.Time `json:"modified_at"`
- Name string `json:"name"`
- Properties map[string]interface{} `json:"properties"`
- UpdatedAt time.Time `json:"updated_at"`
- WritableBy []string `json:"writable_by,omitempty"`
-}
-
-func (g Trait) resourceName() string {
- return "trait"
-}
private Object prefs;
@JsonProperty("writable_by")
private List<String> writableBy;
- @JsonProperty("default_owner_uuid")
- private Boolean defaultOwnerUuid;
public String getEmail() {
return this.email;
return this.writableBy;
}
- public Boolean getDefaultOwnerUuid() {
- return this.defaultOwnerUuid;
- }
-
public void setEmail(String email) {
this.email = email;
}
this.writableBy = writableBy;
}
- public void setDefaultOwnerUuid(Boolean defaultOwnerUuid) {
- this.defaultOwnerUuid = defaultOwnerUuid;
- }
-
public String toString() {
- return "User(email=" + this.getEmail() + ", username=" + this.getUsername() + ", fullName=" + this.getFullName() + ", firstName=" + this.getFirstName() + ", lastName=" + this.getLastName() + ", identityUrl=" + this.getIdentityUrl() + ", isActive=" + this.getIsActive() + ", isAdmin=" + this.getIsAdmin() + ", isInvited=" + this.getIsInvited() + ", prefs=" + this.getPrefs() + ", writableBy=" + this.getWritableBy() + ", defaultOwnerUuid=" + this.getDefaultOwnerUuid() + ")";
+ return "User(email=" + this.getEmail() + ", username=" + this.getUsername() + ", fullName=" + this.getFullName() + ", firstName=" + this.getFirstName() + ", lastName=" + this.getLastName() + ", identityUrl=" + this.getIdentityUrl() + ", isActive=" + this.getIsActive() + ", isAdmin=" + this.getIsAdmin() + ", isInvited=" + this.getIsInvited() + ", prefs=" + this.getPrefs() + ", writableBy=" + this.getWritableBy() + ")";
}
}
"description": "The UUID of the ApiClient in question.",
"required": true,
"location": "path"
+ },
+ "select": {
+ "type": "array",
+ "description": "Attributes of the object to return in the response.",
+ "required": false,
+ "location": "query"
}
},
"parameterOrder": [
"https://api.arvados.org/auth/arvados.readonly"
]
},
- "index": {
+ "list": {
"id": "arvados.api_clients.list",
"path": "api_clients",
"httpMethod": "GET",
"scopes": [
"https://api.arvados.org/auth/arvados"
]
- },
- "list": {
- "id": "arvados.api_clients.list",
- "path": "api_clients",
- "httpMethod": "GET",
- "description": "List ApiClients.\n\n The <code>list</code> method returns a\n <a href=\"/api/resources.html\">resource list</a> of\n matching ApiClients. For example:\n\n <pre>\n {\n \"kind\":\"arvados#apiClientList\",\n \"etag\":\"\",\n \"self_link\":\"\",\n \"next_page_token\":\"\",\n \"next_link\":\"\",\n \"items\":[\n ...\n ],\n \"items_available\":745,\n \"_profile\":{\n \"request_time\":0.157236317\n }\n </pre>",
- "parameters": {
- "filters": {
- "type": "array",
- "required": false,
- "description": "",
- "location": "query"
- },
- "where": {
- "type": "object",
- "required": false,
- "description": "",
- "location": "query"
- },
- "order": {
- "type": "array",
- "required": false,
- "description": "",
- "location": "query"
- },
- "select": {
- "type": "array",
- "description": "Attributes of each object to return in the response.",
- "required": false,
- "location": "query"
- },
- "distinct": {
- "type": "boolean",
- "required": false,
- "default": "false",
- "description": "",
- "location": "query"
- },
- "limit": {
- "type": "integer",
- "required": false,
- "default": "100",
- "description": "",
- "location": "query"
- },
- "offset": {
- "type": "integer",
- "required": false,
- "default": "0",
- "description": "",
- "location": "query"
- },
- "count": {
- "type": "string",
- "required": false,
- "default": "exact",
- "description": "",
- "location": "query"
- },
- "cluster_id": {
- "type": "string",
- "description": "List objects on a remote federated cluster instead of the current one.",
- "location": "query",
- "required": false
- },
- "bypass_federation": {
- "type": "boolean",
- "required": false,
- "description": "bypass federation behavior, list items from local instance database only",
- "location": "query"
- }
- },
- "response": {
- "$ref": "ApiClientList"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados",
- "https://api.arvados.org/auth/arvados.readonly"
- ]
- },
- "show": {
- "id": "arvados.api_clients.show",
- "path": "api_clients/{uuid}",
- "httpMethod": "GET",
- "description": "show api_clients",
- "parameters": {
- "uuid": {
- "type": "string",
- "description": "",
- "required": true,
- "location": "path"
- },
- "select": {
- "type": "array",
- "description": "Attributes of the object to return in the response.",
- "required": false,
- "location": "query"
- }
- },
- "response": {
- "$ref": "ApiClient"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados"
- ]
- },
- "destroy": {
- "id": "arvados.api_clients.destroy",
- "path": "api_clients/{uuid}",
- "httpMethod": "DELETE",
- "description": "destroy api_clients",
- "parameters": {
- "uuid": {
- "type": "string",
- "description": "",
- "required": true,
- "location": "path"
- }
- },
- "response": {
- "$ref": "ApiClient"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados"
- ]
}
}
},
"description": "The UUID of the ApiClientAuthorization in question.",
"required": true,
"location": "path"
+ },
+ "select": {
+ "type": "array",
+ "description": "Attributes of the object to return in the response.",
+ "required": false,
+ "location": "query"
}
},
"parameterOrder": [
"https://api.arvados.org/auth/arvados.readonly"
]
},
- "index": {
+ "list": {
"id": "arvados.api_client_authorizations.list",
"path": "api_client_authorizations",
"httpMethod": "GET",
"scopes": [
"https://api.arvados.org/auth/arvados"
]
+ }
+ }
+ },
+ "authorized_keys": {
+ "methods": {
+ "get": {
+ "id": "arvados.authorized_keys.get",
+ "path": "authorized_keys/{uuid}",
+ "httpMethod": "GET",
+ "description": "Gets a AuthorizedKey's metadata by UUID.",
+ "parameters": {
+ "uuid": {
+ "type": "string",
+ "description": "The UUID of the AuthorizedKey in question.",
+ "required": true,
+ "location": "path"
+ },
+ "select": {
+ "type": "array",
+ "description": "Attributes of the object to return in the response.",
+ "required": false,
+ "location": "query"
+ }
+ },
+ "parameterOrder": [
+ "uuid"
+ ],
+ "response": {
+ "$ref": "AuthorizedKey"
+ },
+ "scopes": [
+ "https://api.arvados.org/auth/arvados",
+ "https://api.arvados.org/auth/arvados.readonly"
+ ]
},
"list": {
- "id": "arvados.api_client_authorizations.list",
- "path": "api_client_authorizations",
+ "id": "arvados.authorized_keys.list",
+ "path": "authorized_keys",
"httpMethod": "GET",
- "description": "List ApiClientAuthorizations.\n\n The <code>list</code> method returns a\n <a href=\"/api/resources.html\">resource list</a> of\n matching ApiClientAuthorizations. For example:\n\n <pre>\n {\n \"kind\":\"arvados#apiClientAuthorizationList\",\n \"etag\":\"\",\n \"self_link\":\"\",\n \"next_page_token\":\"\",\n \"next_link\":\"\",\n \"items\":[\n ...\n ],\n \"items_available\":745,\n \"_profile\":{\n \"request_time\":0.157236317\n }\n </pre>",
+ "description": "List AuthorizedKeys.\n\n The <code>list</code> method returns a\n <a href=\"/api/resources.html\">resource list</a> of\n matching AuthorizedKeys. For example:\n\n <pre>\n {\n \"kind\":\"arvados#authorizedKeyList\",\n \"etag\":\"\",\n \"self_link\":\"\",\n \"next_page_token\":\"\",\n \"next_link\":\"\",\n \"items\":[\n ...\n ],\n \"items_available\":745,\n \"_profile\":{\n \"request_time\":0.157236317\n }\n </pre>",
"parameters": {
"filters": {
"type": "array",
}
},
"response": {
- "$ref": "ApiClientAuthorizationList"
+ "$ref": "AuthorizedKeyList"
},
"scopes": [
"https://api.arvados.org/auth/arvados",
"https://api.arvados.org/auth/arvados.readonly"
]
},
- "show": {
- "id": "arvados.api_client_authorizations.show",
- "path": "api_client_authorizations/{uuid}",
- "httpMethod": "GET",
- "description": "show api_client_authorizations",
+ "create": {
+ "id": "arvados.authorized_keys.create",
+ "path": "authorized_keys",
+ "httpMethod": "POST",
+ "description": "Create a new AuthorizedKey.",
+ "parameters": {
+ "select": {
+ "type": "array",
+ "description": "Attributes of the new object to return in the response.",
+ "required": false,
+ "location": "query"
+ },
+ "ensure_unique_name": {
+ "type": "boolean",
+ "description": "Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision.",
+ "location": "query",
+ "required": false,
+ "default": "false"
+ },
+ "cluster_id": {
+ "type": "string",
+ "description": "Create object on a remote federated cluster instead of the current one.",
+ "location": "query",
+ "required": false
+ }
+ },
+ "request": {
+ "required": true,
+ "properties": {
+ "authorized_key": {
+ "$ref": "AuthorizedKey"
+ }
+ }
+ },
+ "response": {
+ "$ref": "AuthorizedKey"
+ },
+ "scopes": [
+ "https://api.arvados.org/auth/arvados"
+ ]
+ },
+ "update": {
+ "id": "arvados.authorized_keys.update",
+ "path": "authorized_keys/{uuid}",
+ "httpMethod": "PUT",
+ "description": "Update attributes of an existing AuthorizedKey.",
"parameters": {
"uuid": {
"type": "string",
- "description": "",
+ "description": "The UUID of the AuthorizedKey in question.",
"required": true,
"location": "path"
},
"select": {
"type": "array",
- "description": "Attributes of the object to return in the response.",
+ "description": "Attributes of the updated object to return in the response.",
"required": false,
"location": "query"
}
},
+ "request": {
+ "required": true,
+ "properties": {
+ "authorized_key": {
+ "$ref": "AuthorizedKey"
+ }
+ }
+ },
"response": {
- "$ref": "ApiClientAuthorization"
+ "$ref": "AuthorizedKey"
},
"scopes": [
"https://api.arvados.org/auth/arvados"
]
},
- "destroy": {
- "id": "arvados.api_client_authorizations.destroy",
- "path": "api_client_authorizations/{uuid}",
+ "delete": {
+ "id": "arvados.authorized_keys.delete",
+ "path": "authorized_keys/{uuid}",
"httpMethod": "DELETE",
- "description": "destroy api_client_authorizations",
+ "description": "Delete an existing AuthorizedKey.",
"parameters": {
"uuid": {
"type": "string",
- "description": "",
+ "description": "The UUID of the AuthorizedKey in question.",
"required": true,
"location": "path"
}
},
"response": {
- "$ref": "ApiClientAuthorization"
+ "$ref": "AuthorizedKey"
},
"scopes": [
"https://api.arvados.org/auth/arvados"
}
}
},
- "authorized_keys": {
+ "collections": {
"methods": {
"get": {
- "id": "arvados.authorized_keys.get",
- "path": "authorized_keys/{uuid}",
+ "id": "arvados.collections.get",
+ "path": "collections/{uuid}",
"httpMethod": "GET",
- "description": "Gets a AuthorizedKey's metadata by UUID.",
+ "description": "Gets a Collection's metadata by UUID.",
"parameters": {
"uuid": {
"type": "string",
- "description": "The UUID of the AuthorizedKey in question.",
+ "description": "The UUID of the Collection in question.",
"required": true,
"location": "path"
+ },
+ "select": {
+ "type": "array",
+ "description": "Attributes of the object to return in the response.",
+ "required": false,
+ "location": "query"
+ },
+ "include_trash": {
+ "type": "boolean",
+ "required": false,
+ "default": "false",
+ "description": "Show collection even if its is_trashed attribute is true.",
+ "location": "query"
+ },
+ "include_old_versions": {
+ "type": "boolean",
+ "required": false,
+ "default": "true",
+ "description": "Include past collection versions.",
+ "location": "query"
}
},
"parameterOrder": [
"uuid"
],
"response": {
- "$ref": "AuthorizedKey"
+ "$ref": "Collection"
},
"scopes": [
"https://api.arvados.org/auth/arvados",
"https://api.arvados.org/auth/arvados.readonly"
]
},
- "index": {
- "id": "arvados.authorized_keys.list",
- "path": "authorized_keys",
+ "list": {
+ "id": "arvados.collections.list",
+ "path": "collections",
"httpMethod": "GET",
- "description": "List AuthorizedKeys.\n\n The <code>list</code> method returns a\n <a href=\"/api/resources.html\">resource list</a> of\n matching AuthorizedKeys. For example:\n\n <pre>\n {\n \"kind\":\"arvados#authorizedKeyList\",\n \"etag\":\"\",\n \"self_link\":\"\",\n \"next_page_token\":\"\",\n \"next_link\":\"\",\n \"items\":[\n ...\n ],\n \"items_available\":745,\n \"_profile\":{\n \"request_time\":0.157236317\n }\n </pre>",
+ "description": "List Collections.\n\n The <code>list</code> method returns a\n <a href=\"/api/resources.html\">resource list</a> of\n matching Collections. For example:\n\n <pre>\n {\n \"kind\":\"arvados#collectionList\",\n \"etag\":\"\",\n \"self_link\":\"\",\n \"next_page_token\":\"\",\n \"next_link\":\"\",\n \"items\":[\n ...\n ],\n \"items_available\":745,\n \"_profile\":{\n \"request_time\":0.157236317\n }\n </pre>",
"parameters": {
"filters": {
"type": "array",
"required": false,
"description": "bypass federation behavior, list items from local instance database only",
"location": "query"
+ },
+ "include_trash": {
+ "type": "boolean",
+ "required": false,
+ "default": "false",
+ "description": "Include collections whose is_trashed attribute is true.",
+ "location": "query"
+ },
+ "include_old_versions": {
+ "type": "boolean",
+ "required": false,
+ "default": "false",
+ "description": "Include past collection versions.",
+ "location": "query"
}
},
"response": {
- "$ref": "AuthorizedKeyList"
+ "$ref": "CollectionList"
},
"scopes": [
"https://api.arvados.org/auth/arvados",
]
},
"create": {
- "id": "arvados.authorized_keys.create",
- "path": "authorized_keys",
+ "id": "arvados.collections.create",
+ "path": "collections",
"httpMethod": "POST",
- "description": "Create a new AuthorizedKey.",
+ "description": "Create a new Collection.",
"parameters": {
"select": {
"type": "array",
"description": "Create object on a remote federated cluster instead of the current one.",
"location": "query",
"required": false
+ },
+ "replace_files": {
+ "type": "object",
+ "description": "Files and directories to initialize/replace with content from other collections.",
+ "required": false,
+ "location": "query",
+ "properties": {},
+ "additionalProperties": {
+ "type": "string"
+ }
}
},
"request": {
"required": true,
"properties": {
- "authorized_key": {
- "$ref": "AuthorizedKey"
+ "collection": {
+ "$ref": "Collection"
}
}
},
"response": {
- "$ref": "AuthorizedKey"
+ "$ref": "Collection"
},
"scopes": [
"https://api.arvados.org/auth/arvados"
]
},
"update": {
- "id": "arvados.authorized_keys.update",
- "path": "authorized_keys/{uuid}",
+ "id": "arvados.collections.update",
+ "path": "collections/{uuid}",
"httpMethod": "PUT",
- "description": "Update attributes of an existing AuthorizedKey.",
+ "description": "Update attributes of an existing Collection.",
"parameters": {
"uuid": {
"type": "string",
- "description": "The UUID of the AuthorizedKey in question.",
+ "description": "The UUID of the Collection in question.",
"required": true,
"location": "path"
},
"description": "Attributes of the updated object to return in the response.",
"required": false,
"location": "query"
+ },
+ "replace_files": {
+ "type": "object",
+ "description": "Files and directories to initialize/replace with content from other collections.",
+ "required": false,
+ "location": "query",
+ "properties": {},
+ "additionalProperties": {
+ "type": "string"
+ }
}
},
"request": {
"required": true,
"properties": {
- "authorized_key": {
- "$ref": "AuthorizedKey"
+ "collection": {
+ "$ref": "Collection"
}
}
},
"response": {
- "$ref": "AuthorizedKey"
+ "$ref": "Collection"
},
"scopes": [
"https://api.arvados.org/auth/arvados"
]
},
"delete": {
- "id": "arvados.authorized_keys.delete",
- "path": "authorized_keys/{uuid}",
+ "id": "arvados.collections.delete",
+ "path": "collections/{uuid}",
"httpMethod": "DELETE",
- "description": "Delete an existing AuthorizedKey.",
+ "description": "Delete an existing Collection.",
"parameters": {
"uuid": {
"type": "string",
- "description": "The UUID of the AuthorizedKey in question.",
+ "description": "The UUID of the Collection in question.",
"required": true,
"location": "path"
}
},
"response": {
- "$ref": "AuthorizedKey"
+ "$ref": "Collection"
},
"scopes": [
"https://api.arvados.org/auth/arvados"
]
},
- "list": {
- "id": "arvados.authorized_keys.list",
- "path": "authorized_keys",
+ "provenance": {
+ "id": "arvados.collections.provenance",
+ "path": "collections/{uuid}/provenance",
"httpMethod": "GET",
- "description": "List AuthorizedKeys.\n\n The <code>list</code> method returns a\n <a href=\"/api/resources.html\">resource list</a> of\n matching AuthorizedKeys. For example:\n\n <pre>\n {\n \"kind\":\"arvados#authorizedKeyList\",\n \"etag\":\"\",\n \"self_link\":\"\",\n \"next_page_token\":\"\",\n \"next_link\":\"\",\n \"items\":[\n ...\n ],\n \"items_available\":745,\n \"_profile\":{\n \"request_time\":0.157236317\n }\n </pre>",
+ "description": "provenance collections",
"parameters": {
- "filters": {
- "type": "array",
- "required": false,
- "description": "",
- "location": "query"
- },
- "where": {
- "type": "object",
- "required": false,
- "description": "",
- "location": "query"
- },
- "order": {
- "type": "array",
- "required": false,
- "description": "",
- "location": "query"
- },
- "select": {
- "type": "array",
- "description": "Attributes of each object to return in the response.",
- "required": false,
- "location": "query"
- },
- "distinct": {
- "type": "boolean",
- "required": false,
- "default": "false",
- "description": "",
- "location": "query"
- },
- "limit": {
- "type": "integer",
- "required": false,
- "default": "100",
- "description": "",
- "location": "query"
- },
- "offset": {
- "type": "integer",
- "required": false,
- "default": "0",
- "description": "",
- "location": "query"
- },
- "count": {
+ "uuid": {
"type": "string",
- "required": false,
- "default": "exact",
"description": "",
- "location": "query"
- },
- "cluster_id": {
- "type": "string",
- "description": "List objects on a remote federated cluster instead of the current one.",
- "location": "query",
- "required": false
- },
- "bypass_federation": {
- "type": "boolean",
- "required": false,
- "description": "bypass federation behavior, list items from local instance database only",
- "location": "query"
+ "required": true,
+ "location": "path"
}
},
"response": {
- "$ref": "AuthorizedKeyList"
+ "$ref": "Collection"
},
"scopes": [
- "https://api.arvados.org/auth/arvados",
- "https://api.arvados.org/auth/arvados.readonly"
+ "https://api.arvados.org/auth/arvados"
]
},
- "show": {
- "id": "arvados.authorized_keys.show",
- "path": "authorized_keys/{uuid}",
+ "used_by": {
+ "id": "arvados.collections.used_by",
+ "path": "collections/{uuid}/used_by",
"httpMethod": "GET",
- "description": "show authorized_keys",
+ "description": "used_by collections",
"parameters": {
"uuid": {
"type": "string",
"description": "",
"required": true,
"location": "path"
- },
- "select": {
- "type": "array",
- "description": "Attributes of the object to return in the response.",
- "required": false,
- "location": "query"
}
},
"response": {
- "$ref": "AuthorizedKey"
+ "$ref": "Collection"
},
"scopes": [
"https://api.arvados.org/auth/arvados"
]
},
- "destroy": {
- "id": "arvados.authorized_keys.destroy",
- "path": "authorized_keys/{uuid}",
- "httpMethod": "DELETE",
- "description": "destroy authorized_keys",
+ "trash": {
+ "id": "arvados.collections.trash",
+ "path": "collections/{uuid}/trash",
+ "httpMethod": "POST",
+ "description": "trash collections",
"parameters": {
"uuid": {
"type": "string",
}
},
"response": {
- "$ref": "AuthorizedKey"
+ "$ref": "Collection"
+ },
+ "scopes": [
+ "https://api.arvados.org/auth/arvados"
+ ]
+ },
+ "untrash": {
+ "id": "arvados.collections.untrash",
+ "path": "collections/{uuid}/untrash",
+ "httpMethod": "POST",
+ "description": "untrash collections",
+ "parameters": {
+ "uuid": {
+ "type": "string",
+ "description": "",
+ "required": true,
+ "location": "path"
+ }
+ },
+ "response": {
+ "$ref": "Collection"
},
"scopes": [
"https://api.arvados.org/auth/arvados"
}
}
},
- "collections": {
+ "containers": {
"methods": {
"get": {
- "id": "arvados.collections.get",
- "path": "collections/{uuid}",
+ "id": "arvados.containers.get",
+ "path": "containers/{uuid}",
"httpMethod": "GET",
- "description": "Gets a Collection's metadata by UUID.",
+ "description": "Gets a Container's metadata by UUID.",
"parameters": {
"uuid": {
"type": "string",
- "description": "The UUID of the Collection in question.",
+ "description": "The UUID of the Container in question.",
"required": true,
"location": "path"
+ },
+ "select": {
+ "type": "array",
+ "description": "Attributes of the object to return in the response.",
+ "required": false,
+ "location": "query"
}
},
"parameterOrder": [
"uuid"
],
"response": {
- "$ref": "Collection"
+ "$ref": "Container"
},
"scopes": [
"https://api.arvados.org/auth/arvados",
"https://api.arvados.org/auth/arvados.readonly"
]
},
- "index": {
- "id": "arvados.collections.list",
- "path": "collections",
+ "list": {
+ "id": "arvados.containers.list",
+ "path": "containers",
"httpMethod": "GET",
- "description": "List Collections.\n\n The <code>list</code> method returns a\n <a href=\"/api/resources.html\">resource list</a> of\n matching Collections. For example:\n\n <pre>\n {\n \"kind\":\"arvados#collectionList\",\n \"etag\":\"\",\n \"self_link\":\"\",\n \"next_page_token\":\"\",\n \"next_link\":\"\",\n \"items\":[\n ...\n ],\n \"items_available\":745,\n \"_profile\":{\n \"request_time\":0.157236317\n }\n </pre>",
+ "description": "List Containers.\n\n The <code>list</code> method returns a\n <a href=\"/api/resources.html\">resource list</a> of\n matching Containers. For example:\n\n <pre>\n {\n \"kind\":\"arvados#containerList\",\n \"etag\":\"\",\n \"self_link\":\"\",\n \"next_page_token\":\"\",\n \"next_link\":\"\",\n \"items\":[\n ...\n ],\n \"items_available\":745,\n \"_profile\":{\n \"request_time\":0.157236317\n }\n </pre>",
"parameters": {
"filters": {
"type": "array",
"required": false,
"description": "bypass federation behavior, list items from local instance database only",
"location": "query"
- },
- "include_trash": {
- "type": "boolean",
- "required": false,
- "default": "false",
- "description": "Include collections whose is_trashed attribute is true.",
- "location": "query"
- },
- "include_old_versions": {
- "type": "boolean",
- "required": false,
- "default": "false",
- "description": "Include past collection versions.",
- "location": "query"
}
},
"response": {
- "$ref": "CollectionList"
+ "$ref": "ContainerList"
},
"scopes": [
"https://api.arvados.org/auth/arvados",
]
},
"create": {
- "id": "arvados.collections.create",
- "path": "collections",
+ "id": "arvados.containers.create",
+ "path": "containers",
"httpMethod": "POST",
- "description": "Create a new Collection.",
+ "description": "Create a new Container.",
"parameters": {
"select": {
"type": "array",
"description": "Create object on a remote federated cluster instead of the current one.",
"location": "query",
"required": false
- },
- "replace_files": {
- "type": "object",
- "description": "Files and directories to initialize/replace with content from other collections.",
- "required": false,
- "location": "query",
- "properties": {},
- "additionalProperties": {
- "type": "string"
- }
}
},
"request": {
"required": true,
"properties": {
- "collection": {
- "$ref": "Collection"
+ "container": {
+ "$ref": "Container"
}
}
},
"response": {
- "$ref": "Collection"
+ "$ref": "Container"
},
"scopes": [
"https://api.arvados.org/auth/arvados"
]
},
"update": {
- "id": "arvados.collections.update",
- "path": "collections/{uuid}",
+ "id": "arvados.containers.update",
+ "path": "containers/{uuid}",
"httpMethod": "PUT",
- "description": "Update attributes of an existing Collection.",
+ "description": "Update attributes of an existing Container.",
"parameters": {
"uuid": {
"type": "string",
- "description": "The UUID of the Collection in question.",
+ "description": "The UUID of the Container in question.",
"required": true,
"location": "path"
},
"description": "Attributes of the updated object to return in the response.",
"required": false,
"location": "query"
- },
- "replace_files": {
- "type": "object",
- "description": "Files and directories to initialize/replace with content from other collections.",
- "required": false,
- "location": "query",
- "properties": {},
- "additionalProperties": {
- "type": "string"
- }
}
},
"request": {
"required": true,
"properties": {
- "collection": {
- "$ref": "Collection"
+ "container": {
+ "$ref": "Container"
}
}
},
"response": {
- "$ref": "Collection"
+ "$ref": "Container"
},
"scopes": [
"https://api.arvados.org/auth/arvados"
]
},
"delete": {
- "id": "arvados.collections.delete",
- "path": "collections/{uuid}",
+ "id": "arvados.containers.delete",
+ "path": "containers/{uuid}",
"httpMethod": "DELETE",
- "description": "Delete an existing Collection.",
+ "description": "Delete an existing Container.",
"parameters": {
"uuid": {
"type": "string",
- "description": "The UUID of the Collection in question.",
+ "description": "The UUID of the Container in question.",
"required": true,
"location": "path"
}
},
"response": {
- "$ref": "Collection"
+ "$ref": "Container"
},
"scopes": [
"https://api.arvados.org/auth/arvados"
]
},
- "provenance": {
- "id": "arvados.collections.provenance",
- "path": "collections/{uuid}/provenance",
+ "auth": {
+ "id": "arvados.containers.auth",
+ "path": "containers/{uuid}/auth",
"httpMethod": "GET",
- "description": "provenance collections",
+ "description": "auth containers",
"parameters": {
"uuid": {
"type": "string",
}
},
"response": {
- "$ref": "Collection"
+ "$ref": "Container"
},
"scopes": [
"https://api.arvados.org/auth/arvados"
]
},
- "used_by": {
- "id": "arvados.collections.used_by",
- "path": "collections/{uuid}/used_by",
- "httpMethod": "GET",
- "description": "used_by collections",
+ "lock": {
+ "id": "arvados.containers.lock",
+ "path": "containers/{uuid}/lock",
+ "httpMethod": "POST",
+ "description": "lock containers",
"parameters": {
"uuid": {
"type": "string",
}
},
"response": {
- "$ref": "Collection"
+ "$ref": "Container"
},
"scopes": [
"https://api.arvados.org/auth/arvados"
]
},
- "trash": {
- "id": "arvados.collections.trash",
- "path": "collections/{uuid}/trash",
+ "unlock": {
+ "id": "arvados.containers.unlock",
+ "path": "containers/{uuid}/unlock",
"httpMethod": "POST",
- "description": "trash collections",
+ "description": "unlock containers",
"parameters": {
"uuid": {
"type": "string",
}
},
"response": {
- "$ref": "Collection"
+ "$ref": "Container"
},
"scopes": [
"https://api.arvados.org/auth/arvados"
]
},
- "untrash": {
- "id": "arvados.collections.untrash",
- "path": "collections/{uuid}/untrash",
+ "update_priority": {
+ "id": "arvados.containers.update_priority",
+ "path": "containers/{uuid}/update_priority",
"httpMethod": "POST",
- "description": "untrash collections",
+ "description": "update_priority containers",
"parameters": {
"uuid": {
"type": "string",
}
},
"response": {
- "$ref": "Collection"
+ "$ref": "Container"
},
"scopes": [
"https://api.arvados.org/auth/arvados"
]
},
- "list": {
- "id": "arvados.collections.list",
- "path": "collections",
- "httpMethod": "GET",
- "description": "List Collections.\n\n The <code>list</code> method returns a\n <a href=\"/api/resources.html\">resource list</a> of\n matching Collections. For example:\n\n <pre>\n {\n \"kind\":\"arvados#collectionList\",\n \"etag\":\"\",\n \"self_link\":\"\",\n \"next_page_token\":\"\",\n \"next_link\":\"\",\n \"items\":[\n ...\n ],\n \"items_available\":745,\n \"_profile\":{\n \"request_time\":0.157236317\n }\n </pre>",
- "parameters": {
- "filters": {
- "type": "array",
- "required": false,
- "description": "",
- "location": "query"
- },
- "where": {
- "type": "object",
- "required": false,
- "description": "",
- "location": "query"
- },
- "order": {
- "type": "array",
- "required": false,
- "description": "",
- "location": "query"
- },
- "select": {
- "type": "array",
- "description": "Attributes of each object to return in the response.",
- "required": false,
- "location": "query"
- },
- "distinct": {
- "type": "boolean",
- "required": false,
- "default": "false",
- "description": "",
- "location": "query"
- },
- "limit": {
- "type": "integer",
- "required": false,
- "default": "100",
- "description": "",
- "location": "query"
- },
- "offset": {
- "type": "integer",
- "required": false,
- "default": "0",
- "description": "",
- "location": "query"
- },
- "count": {
- "type": "string",
- "required": false,
- "default": "exact",
- "description": "",
- "location": "query"
- },
- "cluster_id": {
- "type": "string",
- "description": "List objects on a remote federated cluster instead of the current one.",
- "location": "query",
- "required": false
- },
- "bypass_federation": {
- "type": "boolean",
- "required": false,
- "description": "bypass federation behavior, list items from local instance database only",
- "location": "query"
- },
- "include_trash": {
- "type": "boolean",
- "required": false,
- "default": "false",
- "description": "Include collections whose is_trashed attribute is true.",
- "location": "query"
- },
- "include_old_versions": {
- "type": "boolean",
- "required": false,
- "default": "false",
- "description": "Include past collection versions.",
- "location": "query"
- }
- },
- "response": {
- "$ref": "CollectionList"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados",
- "https://api.arvados.org/auth/arvados.readonly"
- ]
- },
- "show": {
- "id": "arvados.collections.show",
- "path": "collections/{uuid}",
+ "secret_mounts": {
+ "id": "arvados.containers.secret_mounts",
+ "path": "containers/{uuid}/secret_mounts",
"httpMethod": "GET",
- "description": "show collections",
+ "description": "secret_mounts containers",
"parameters": {
"uuid": {
"type": "string",
"description": "",
"required": true,
"location": "path"
- },
- "select": {
- "type": "array",
- "description": "Attributes of the object to return in the response.",
- "required": false,
- "location": "query"
- },
- "include_trash": {
- "type": "boolean",
- "required": false,
- "default": "false",
- "description": "Show collection even if its is_trashed attribute is true.",
- "location": "query"
- },
- "include_old_versions": {
- "type": "boolean",
- "required": false,
- "default": "true",
- "description": "Include past collection versions.",
- "location": "query"
}
},
"response": {
- "$ref": "Collection"
+ "$ref": "Container"
},
"scopes": [
"https://api.arvados.org/auth/arvados"
]
},
- "destroy": {
- "id": "arvados.collections.destroy",
- "path": "collections/{uuid}",
- "httpMethod": "DELETE",
- "description": "destroy collections",
- "parameters": {
- "uuid": {
- "type": "string",
- "description": "",
- "required": true,
- "location": "path"
- }
- },
+ "current": {
+ "id": "arvados.containers.current",
+ "path": "containers/current",
+ "httpMethod": "GET",
+ "description": "current containers",
+ "parameters": {},
"response": {
- "$ref": "Collection"
+ "$ref": "Container"
},
"scopes": [
"https://api.arvados.org/auth/arvados"
]
}
}
- },
- "containers": {
- "methods": {
- "get": {
- "id": "arvados.containers.get",
- "path": "containers/{uuid}",
- "httpMethod": "GET",
- "description": "Gets a Container's metadata by UUID.",
- "parameters": {
- "uuid": {
- "type": "string",
- "description": "The UUID of the Container in question.",
- "required": true,
- "location": "path"
- }
- },
- "parameterOrder": [
- "uuid"
- ],
- "response": {
- "$ref": "Container"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados",
- "https://api.arvados.org/auth/arvados.readonly"
- ]
- },
- "index": {
- "id": "arvados.containers.list",
- "path": "containers",
- "httpMethod": "GET",
- "description": "List Containers.\n\n The <code>list</code> method returns a\n <a href=\"/api/resources.html\">resource list</a> of\n matching Containers. For example:\n\n <pre>\n {\n \"kind\":\"arvados#containerList\",\n \"etag\":\"\",\n \"self_link\":\"\",\n \"next_page_token\":\"\",\n \"next_link\":\"\",\n \"items\":[\n ...\n ],\n \"items_available\":745,\n \"_profile\":{\n \"request_time\":0.157236317\n }\n </pre>",
- "parameters": {
- "filters": {
- "type": "array",
- "required": false,
- "description": "",
- "location": "query"
- },
- "where": {
- "type": "object",
- "required": false,
- "description": "",
- "location": "query"
- },
- "order": {
- "type": "array",
- "required": false,
- "description": "",
- "location": "query"
- },
- "select": {
- "type": "array",
- "description": "Attributes of each object to return in the response.",
- "required": false,
- "location": "query"
- },
- "distinct": {
- "type": "boolean",
- "required": false,
- "default": "false",
- "description": "",
- "location": "query"
- },
- "limit": {
- "type": "integer",
- "required": false,
- "default": "100",
- "description": "",
- "location": "query"
- },
- "offset": {
- "type": "integer",
- "required": false,
- "default": "0",
- "description": "",
- "location": "query"
- },
- "count": {
- "type": "string",
- "required": false,
- "default": "exact",
- "description": "",
- "location": "query"
- },
- "cluster_id": {
- "type": "string",
- "description": "List objects on a remote federated cluster instead of the current one.",
- "location": "query",
- "required": false
- },
- "bypass_federation": {
- "type": "boolean",
- "required": false,
- "description": "bypass federation behavior, list items from local instance database only",
- "location": "query"
- }
- },
- "response": {
- "$ref": "ContainerList"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados",
- "https://api.arvados.org/auth/arvados.readonly"
- ]
- },
- "create": {
- "id": "arvados.containers.create",
- "path": "containers",
- "httpMethod": "POST",
- "description": "Create a new Container.",
- "parameters": {
- "select": {
- "type": "array",
- "description": "Attributes of the new object to return in the response.",
- "required": false,
- "location": "query"
- },
- "ensure_unique_name": {
- "type": "boolean",
- "description": "Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision.",
- "location": "query",
- "required": false,
- "default": "false"
- },
- "cluster_id": {
- "type": "string",
- "description": "Create object on a remote federated cluster instead of the current one.",
- "location": "query",
- "required": false
- }
- },
- "request": {
- "required": true,
- "properties": {
- "container": {
- "$ref": "Container"
- }
- }
- },
- "response": {
- "$ref": "Container"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados"
- ]
- },
- "update": {
- "id": "arvados.containers.update",
- "path": "containers/{uuid}",
- "httpMethod": "PUT",
- "description": "Update attributes of an existing Container.",
- "parameters": {
- "uuid": {
- "type": "string",
- "description": "The UUID of the Container in question.",
- "required": true,
- "location": "path"
- },
- "select": {
- "type": "array",
- "description": "Attributes of the updated object to return in the response.",
- "required": false,
- "location": "query"
- }
- },
- "request": {
- "required": true,
- "properties": {
- "container": {
- "$ref": "Container"
- }
- }
- },
- "response": {
- "$ref": "Container"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados"
- ]
- },
- "delete": {
- "id": "arvados.containers.delete",
- "path": "containers/{uuid}",
- "httpMethod": "DELETE",
- "description": "Delete an existing Container.",
- "parameters": {
- "uuid": {
- "type": "string",
- "description": "The UUID of the Container in question.",
- "required": true,
- "location": "path"
- }
- },
- "response": {
- "$ref": "Container"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados"
- ]
- },
- "auth": {
- "id": "arvados.containers.auth",
- "path": "containers/{uuid}/auth",
- "httpMethod": "GET",
- "description": "auth containers",
- "parameters": {
- "uuid": {
- "type": "string",
- "description": "",
- "required": true,
- "location": "path"
- }
- },
- "response": {
- "$ref": "Container"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados"
- ]
- },
- "lock": {
- "id": "arvados.containers.lock",
- "path": "containers/{uuid}/lock",
- "httpMethod": "POST",
- "description": "lock containers",
- "parameters": {
- "uuid": {
- "type": "string",
- "description": "",
- "required": true,
- "location": "path"
- }
- },
- "response": {
- "$ref": "Container"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados"
- ]
- },
- "unlock": {
- "id": "arvados.containers.unlock",
- "path": "containers/{uuid}/unlock",
- "httpMethod": "POST",
- "description": "unlock containers",
- "parameters": {
- "uuid": {
- "type": "string",
- "description": "",
- "required": true,
- "location": "path"
- }
- },
- "response": {
- "$ref": "Container"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados"
- ]
- },
- "update_priority": {
- "id": "arvados.containers.update_priority",
- "path": "containers/{uuid}/update_priority",
- "httpMethod": "POST",
- "description": "update_priority containers",
- "parameters": {
- "uuid": {
- "type": "string",
- "description": "",
- "required": true,
- "location": "path"
- }
- },
- "response": {
- "$ref": "Container"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados"
- ]
- },
- "secret_mounts": {
- "id": "arvados.containers.secret_mounts",
- "path": "containers/{uuid}/secret_mounts",
- "httpMethod": "GET",
- "description": "secret_mounts containers",
- "parameters": {
- "uuid": {
- "type": "string",
- "description": "",
- "required": true,
- "location": "path"
- }
- },
- "response": {
- "$ref": "Container"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados"
- ]
- },
- "current": {
- "id": "arvados.containers.current",
- "path": "containers/current",
- "httpMethod": "GET",
- "description": "current containers",
- "parameters": {},
- "response": {
- "$ref": "Container"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados"
- ]
- },
- "list": {
- "id": "arvados.containers.list",
- "path": "containers",
- "httpMethod": "GET",
- "description": "List Containers.\n\n The <code>list</code> method returns a\n <a href=\"/api/resources.html\">resource list</a> of\n matching Containers. For example:\n\n <pre>\n {\n \"kind\":\"arvados#containerList\",\n \"etag\":\"\",\n \"self_link\":\"\",\n \"next_page_token\":\"\",\n \"next_link\":\"\",\n \"items\":[\n ...\n ],\n \"items_available\":745,\n \"_profile\":{\n \"request_time\":0.157236317\n }\n </pre>",
- "parameters": {
- "filters": {
- "type": "array",
- "required": false,
- "description": "",
- "location": "query"
- },
- "where": {
- "type": "object",
- "required": false,
- "description": "",
- "location": "query"
- },
- "order": {
- "type": "array",
- "required": false,
- "description": "",
- "location": "query"
- },
- "select": {
- "type": "array",
- "description": "Attributes of each object to return in the response.",
- "required": false,
- "location": "query"
- },
- "distinct": {
- "type": "boolean",
- "required": false,
- "default": "false",
- "description": "",
- "location": "query"
- },
- "limit": {
- "type": "integer",
- "required": false,
- "default": "100",
- "description": "",
- "location": "query"
- },
- "offset": {
- "type": "integer",
- "required": false,
- "default": "0",
- "description": "",
- "location": "query"
- },
- "count": {
- "type": "string",
- "required": false,
- "default": "exact",
- "description": "",
- "location": "query"
- },
- "cluster_id": {
- "type": "string",
- "description": "List objects on a remote federated cluster instead of the current one.",
- "location": "query",
- "required": false
- },
- "bypass_federation": {
- "type": "boolean",
- "required": false,
- "description": "bypass federation behavior, list items from local instance database only",
- "location": "query"
- }
- },
- "response": {
- "$ref": "ContainerList"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados",
- "https://api.arvados.org/auth/arvados.readonly"
- ]
- },
- "show": {
- "id": "arvados.containers.show",
- "path": "containers/{uuid}",
- "httpMethod": "GET",
- "description": "show containers",
- "parameters": {
- "uuid": {
- "type": "string",
- "description": "",
- "required": true,
- "location": "path"
- },
- "select": {
- "type": "array",
- "description": "Attributes of the object to return in the response.",
- "required": false,
- "location": "query"
- }
- },
- "response": {
- "$ref": "Container"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados"
- ]
- },
- "destroy": {
- "id": "arvados.containers.destroy",
- "path": "containers/{uuid}",
- "httpMethod": "DELETE",
- "description": "destroy containers",
- "parameters": {
- "uuid": {
- "type": "string",
- "description": "",
- "required": true,
- "location": "path"
- }
- },
- "response": {
- "$ref": "Container"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados"
- ]
- }
- }
- },
- "container_requests": {
- "methods": {
- "get": {
- "id": "arvados.container_requests.get",
- "path": "container_requests/{uuid}",
- "httpMethod": "GET",
- "description": "Gets a ContainerRequest's metadata by UUID.",
- "parameters": {
- "uuid": {
- "type": "string",
- "description": "The UUID of the ContainerRequest in question.",
- "required": true,
- "location": "path"
- }
- },
- "parameterOrder": [
- "uuid"
- ],
- "response": {
- "$ref": "ContainerRequest"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados",
- "https://api.arvados.org/auth/arvados.readonly"
- ]
- },
- "index": {
- "id": "arvados.container_requests.list",
- "path": "container_requests",
- "httpMethod": "GET",
- "description": "List ContainerRequests.\n\n The <code>list</code> method returns a\n <a href=\"/api/resources.html\">resource list</a> of\n matching ContainerRequests. For example:\n\n <pre>\n {\n \"kind\":\"arvados#containerRequestList\",\n \"etag\":\"\",\n \"self_link\":\"\",\n \"next_page_token\":\"\",\n \"next_link\":\"\",\n \"items\":[\n ...\n ],\n \"items_available\":745,\n \"_profile\":{\n \"request_time\":0.157236317\n }\n </pre>",
- "parameters": {
- "filters": {
- "type": "array",
- "required": false,
- "description": "",
- "location": "query"
- },
- "where": {
- "type": "object",
- "required": false,
- "description": "",
- "location": "query"
- },
- "order": {
- "type": "array",
- "required": false,
- "description": "",
- "location": "query"
- },
- "select": {
- "type": "array",
- "description": "Attributes of each object to return in the response.",
- "required": false,
- "location": "query"
- },
- "distinct": {
- "type": "boolean",
- "required": false,
- "default": "false",
- "description": "",
- "location": "query"
- },
- "limit": {
- "type": "integer",
- "required": false,
- "default": "100",
- "description": "",
- "location": "query"
- },
- "offset": {
- "type": "integer",
- "required": false,
- "default": "0",
- "description": "",
- "location": "query"
- },
- "count": {
- "type": "string",
- "required": false,
- "default": "exact",
- "description": "",
- "location": "query"
- },
- "cluster_id": {
- "type": "string",
- "description": "List objects on a remote federated cluster instead of the current one.",
- "location": "query",
- "required": false
- },
- "bypass_federation": {
- "type": "boolean",
- "required": false,
- "description": "bypass federation behavior, list items from local instance database only",
- "location": "query"
- },
- "include_trash": {
- "type": "boolean",
- "required": false,
- "default": "false",
- "description": "Include container requests whose owner project is trashed.",
- "location": "query"
- }
- },
- "response": {
- "$ref": "ContainerRequestList"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados",
- "https://api.arvados.org/auth/arvados.readonly"
- ]
- },
- "create": {
- "id": "arvados.container_requests.create",
- "path": "container_requests",
- "httpMethod": "POST",
- "description": "Create a new ContainerRequest.",
- "parameters": {
- "select": {
- "type": "array",
- "description": "Attributes of the new object to return in the response.",
- "required": false,
- "location": "query"
- },
- "ensure_unique_name": {
- "type": "boolean",
- "description": "Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision.",
- "location": "query",
- "required": false,
- "default": "false"
- },
- "cluster_id": {
- "type": "string",
- "description": "Create object on a remote federated cluster instead of the current one.",
- "location": "query",
- "required": false
- }
- },
- "request": {
- "required": true,
- "properties": {
- "container_request": {
- "$ref": "ContainerRequest"
- }
- }
- },
- "response": {
- "$ref": "ContainerRequest"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados"
- ]
- },
- "update": {
- "id": "arvados.container_requests.update",
- "path": "container_requests/{uuid}",
- "httpMethod": "PUT",
- "description": "Update attributes of an existing ContainerRequest.",
- "parameters": {
- "uuid": {
- "type": "string",
- "description": "The UUID of the ContainerRequest in question.",
- "required": true,
- "location": "path"
- },
- "select": {
- "type": "array",
- "description": "Attributes of the updated object to return in the response.",
- "required": false,
- "location": "query"
- }
- },
- "request": {
- "required": true,
- "properties": {
- "container_request": {
- "$ref": "ContainerRequest"
- }
- }
- },
- "response": {
- "$ref": "ContainerRequest"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados"
- ]
- },
- "delete": {
- "id": "arvados.container_requests.delete",
- "path": "container_requests/{uuid}",
- "httpMethod": "DELETE",
- "description": "Delete an existing ContainerRequest.",
- "parameters": {
- "uuid": {
- "type": "string",
- "description": "The UUID of the ContainerRequest in question.",
- "required": true,
- "location": "path"
- }
- },
- "response": {
- "$ref": "ContainerRequest"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados"
- ]
- },
- "container_status": {
- "id": "arvados.container_requests.container_status",
- "path": "container_requests/{uuid}/container_status",
+ },
+ "container_requests": {
+ "methods": {
+ "get": {
+ "id": "arvados.container_requests.get",
+ "path": "container_requests/{uuid}",
"httpMethod": "GET",
- "description": "container_status container_requests",
+ "description": "Gets a ContainerRequest's metadata by UUID.",
"parameters": {
"uuid": {
"type": "string",
- "required": true,
"description": "The UUID of the ContainerRequest in question.",
+ "required": true,
+ "location": "path"
+ },
+ "select": {
+ "type": "array",
+ "description": "Attributes of the object to return in the response.",
+ "required": false,
+ "location": "query"
+ },
+ "include_trash": {
+ "type": "boolean",
+ "required": false,
+ "default": "false",
+ "description": "Show container request even if its owner project is trashed.",
"location": "query"
}
},
+ "parameterOrder": [
+ "uuid"
+ ],
"response": {
"$ref": "ContainerRequest"
},
"scopes": [
- "https://api.arvados.org/auth/arvados"
+ "https://api.arvados.org/auth/arvados",
+ "https://api.arvados.org/auth/arvados.readonly"
]
},
"list": {
"https://api.arvados.org/auth/arvados.readonly"
]
},
- "show": {
- "id": "arvados.container_requests.show",
+ "create": {
+ "id": "arvados.container_requests.create",
+ "path": "container_requests",
+ "httpMethod": "POST",
+ "description": "Create a new ContainerRequest.",
+ "parameters": {
+ "select": {
+ "type": "array",
+ "description": "Attributes of the new object to return in the response.",
+ "required": false,
+ "location": "query"
+ },
+ "ensure_unique_name": {
+ "type": "boolean",
+ "description": "Adjust name to ensure uniqueness instead of returning an error on (owner_uuid, name) collision.",
+ "location": "query",
+ "required": false,
+ "default": "false"
+ },
+ "cluster_id": {
+ "type": "string",
+ "description": "Create object on a remote federated cluster instead of the current one.",
+ "location": "query",
+ "required": false
+ }
+ },
+ "request": {
+ "required": true,
+ "properties": {
+ "container_request": {
+ "$ref": "ContainerRequest"
+ }
+ }
+ },
+ "response": {
+ "$ref": "ContainerRequest"
+ },
+ "scopes": [
+ "https://api.arvados.org/auth/arvados"
+ ]
+ },
+ "update": {
+ "id": "arvados.container_requests.update",
"path": "container_requests/{uuid}",
- "httpMethod": "GET",
- "description": "show container_requests",
+ "httpMethod": "PUT",
+ "description": "Update attributes of an existing ContainerRequest.",
"parameters": {
"uuid": {
"type": "string",
- "description": "",
+ "description": "The UUID of the ContainerRequest in question.",
"required": true,
"location": "path"
},
"select": {
"type": "array",
- "description": "Attributes of the object to return in the response.",
- "required": false,
- "location": "query"
- },
- "include_trash": {
- "type": "boolean",
+ "description": "Attributes of the updated object to return in the response.",
"required": false,
- "default": "false",
- "description": "Show container request even if its owner project is trashed.",
"location": "query"
}
},
+ "request": {
+ "required": true,
+ "properties": {
+ "container_request": {
+ "$ref": "ContainerRequest"
+ }
+ }
+ },
"response": {
"$ref": "ContainerRequest"
},
"https://api.arvados.org/auth/arvados"
]
},
- "destroy": {
- "id": "arvados.container_requests.destroy",
+ "delete": {
+ "id": "arvados.container_requests.delete",
"path": "container_requests/{uuid}",
"httpMethod": "DELETE",
- "description": "destroy container_requests",
+ "description": "Delete an existing ContainerRequest.",
"parameters": {
"uuid": {
"type": "string",
- "description": "",
+ "description": "The UUID of the ContainerRequest in question.",
"required": true,
"location": "path"
}
"scopes": [
"https://api.arvados.org/auth/arvados"
]
+ },
+ "container_status": {
+ "id": "arvados.container_requests.container_status",
+ "path": "container_requests/{uuid}/container_status",
+ "httpMethod": "GET",
+ "description": "container_status container_requests",
+ "parameters": {
+ "uuid": {
+ "type": "string",
+ "required": true,
+ "description": "The UUID of the ContainerRequest in question.",
+ "location": "query"
+ }
+ },
+ "response": {
+ "$ref": "ContainerRequest"
+ },
+ "scopes": [
+ "https://api.arvados.org/auth/arvados"
+ ]
}
}
},
"description": "The UUID of the Group in question.",
"required": true,
"location": "path"
+ },
+ "select": {
+ "type": "array",
+ "description": "Attributes of the object to return in the response.",
+ "required": false,
+ "location": "query"
+ },
+ "include_trash": {
+ "type": "boolean",
+ "required": false,
+ "default": "false",
+ "description": "Show group/project even if its is_trashed attribute is true.",
+ "location": "query"
}
},
"parameterOrder": [
"https://api.arvados.org/auth/arvados.readonly"
]
},
- "index": {
+ "list": {
"id": "arvados.groups.list",
"path": "groups",
"httpMethod": "GET",
"https://api.arvados.org/auth/arvados"
]
},
- "delete": {
- "id": "arvados.groups.delete",
- "path": "groups/{uuid}",
- "httpMethod": "DELETE",
- "description": "Delete an existing Group.",
- "parameters": {
- "uuid": {
- "type": "string",
- "description": "The UUID of the Group in question.",
- "required": true,
- "location": "path"
- }
- },
- "response": {
- "$ref": "Group"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados"
- ]
- },
- "contents": {
- "id": "arvados.groups.contents",
- "path": "groups/contents",
- "httpMethod": "GET",
- "description": "contents groups",
- "parameters": {
- "filters": {
- "type": "array",
- "required": false,
- "description": "",
- "location": "query"
- },
- "where": {
- "type": "object",
- "required": false,
- "description": "",
- "location": "query"
- },
- "order": {
- "type": "array",
- "required": false,
- "description": "",
- "location": "query"
- },
- "select": {
- "type": "array",
- "description": "Attributes of each object to return in the response.",
- "required": false,
- "location": "query"
- },
- "distinct": {
- "type": "boolean",
- "required": false,
- "default": "false",
- "description": "",
- "location": "query"
- },
- "limit": {
- "type": "integer",
- "required": false,
- "default": "100",
- "description": "",
- "location": "query"
- },
- "offset": {
- "type": "integer",
- "required": false,
- "default": "0",
- "description": "",
- "location": "query"
- },
- "count": {
- "type": "string",
- "required": false,
- "default": "exact",
- "description": "",
- "location": "query"
- },
- "cluster_id": {
- "type": "string",
- "description": "List objects on a remote federated cluster instead of the current one.",
- "location": "query",
- "required": false
- },
- "bypass_federation": {
- "type": "boolean",
- "required": false,
- "description": "bypass federation behavior, list items from local instance database only",
- "location": "query"
- },
- "include_trash": {
- "type": "boolean",
- "required": false,
- "default": "false",
- "description": "Include items whose is_trashed attribute is true.",
- "location": "query"
- },
- "uuid": {
- "type": "string",
- "required": false,
- "default": "",
- "description": "",
- "location": "query"
- },
- "recursive": {
- "type": "boolean",
- "required": false,
- "default": "false",
- "description": "Include contents from child groups recursively.",
- "location": "query"
- },
- "include": {
- "type": "array",
- "required": false,
- "description": "Include objects referred to by listed fields in \"included\" response field. Subsets of [\"owner_uuid\", \"container_uuid\"] are supported.",
- "location": "query"
- },
- "include_old_versions": {
- "type": "boolean",
- "required": false,
- "default": "false",
- "description": "Include past collection versions.",
- "location": "query"
+ "delete": {
+ "id": "arvados.groups.delete",
+ "path": "groups/{uuid}",
+ "httpMethod": "DELETE",
+ "description": "Delete an existing Group.",
+ "parameters": {
+ "uuid": {
+ "type": "string",
+ "description": "The UUID of the Group in question.",
+ "required": true,
+ "location": "path"
}
},
"response": {
"https://api.arvados.org/auth/arvados"
]
},
- "shared": {
- "id": "arvados.groups.shared",
- "path": "groups/shared",
+ "contents": {
+ "id": "arvados.groups.contents",
+ "path": "groups/contents",
"httpMethod": "GET",
- "description": "shared groups",
+ "description": "contents groups",
"parameters": {
"filters": {
"type": "array",
"description": "Include items whose is_trashed attribute is true.",
"location": "query"
},
- "include": {
+ "uuid": {
"type": "string",
"required": false,
+ "default": "",
"description": "",
"location": "query"
+ },
+ "recursive": {
+ "type": "boolean",
+ "required": false,
+ "default": "false",
+ "description": "Include contents from child groups recursively.",
+ "location": "query"
+ },
+ "include": {
+ "type": "array",
+ "required": false,
+ "description": "Include objects referred to by listed fields in \"included\" response field. Subsets of [\"owner_uuid\", \"container_uuid\"] are supported.",
+ "location": "query"
+ },
+ "include_old_versions": {
+ "type": "boolean",
+ "required": false,
+ "default": "false",
+ "description": "Include past collection versions.",
+ "location": "query"
}
},
"response": {
"https://api.arvados.org/auth/arvados"
]
},
- "trash": {
- "id": "arvados.groups.trash",
- "path": "groups/{uuid}/trash",
- "httpMethod": "POST",
- "description": "trash groups",
- "parameters": {
- "uuid": {
- "type": "string",
- "description": "",
- "required": true,
- "location": "path"
- }
- },
- "response": {
- "$ref": "Group"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados"
- ]
- },
- "untrash": {
- "id": "arvados.groups.untrash",
- "path": "groups/{uuid}/untrash",
- "httpMethod": "POST",
- "description": "untrash groups",
- "parameters": {
- "uuid": {
- "type": "string",
- "description": "",
- "required": true,
- "location": "path"
- }
- },
- "response": {
- "$ref": "Group"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados"
- ]
- },
- "list": {
- "id": "arvados.groups.list",
- "path": "groups",
+ "shared": {
+ "id": "arvados.groups.shared",
+ "path": "groups/shared",
"httpMethod": "GET",
- "description": "List Groups.\n\n The <code>list</code> method returns a\n <a href=\"/api/resources.html\">resource list</a> of\n matching Groups. For example:\n\n <pre>\n {\n \"kind\":\"arvados#groupList\",\n \"etag\":\"\",\n \"self_link\":\"\",\n \"next_page_token\":\"\",\n \"next_link\":\"\",\n \"items\":[\n ...\n ],\n \"items_available\":745,\n \"_profile\":{\n \"request_time\":0.157236317\n }\n </pre>",
+ "description": "shared groups",
"parameters": {
"filters": {
"type": "array",
"default": "false",
"description": "Include items whose is_trashed attribute is true.",
"location": "query"
+ },
+ "include": {
+ "type": "string",
+ "required": false,
+ "description": "",
+ "location": "query"
}
},
"response": {
- "$ref": "GroupList"
+ "$ref": "Group"
},
"scopes": [
- "https://api.arvados.org/auth/arvados",
- "https://api.arvados.org/auth/arvados.readonly"
+ "https://api.arvados.org/auth/arvados"
]
},
- "show": {
- "id": "arvados.groups.show",
- "path": "groups/{uuid}",
- "httpMethod": "GET",
- "description": "show groups",
+ "trash": {
+ "id": "arvados.groups.trash",
+ "path": "groups/{uuid}/trash",
+ "httpMethod": "POST",
+ "description": "trash groups",
"parameters": {
"uuid": {
"type": "string",
"description": "",
"required": true,
"location": "path"
- },
- "select": {
- "type": "array",
- "description": "Attributes of the object to return in the response.",
- "required": false,
- "location": "query"
- },
- "include_trash": {
- "type": "boolean",
- "required": false,
- "default": "false",
- "description": "Show group/project even if its is_trashed attribute is true.",
- "location": "query"
}
},
"response": {
"https://api.arvados.org/auth/arvados"
]
},
- "destroy": {
- "id": "arvados.groups.destroy",
- "path": "groups/{uuid}",
- "httpMethod": "DELETE",
- "description": "destroy groups",
+ "untrash": {
+ "id": "arvados.groups.untrash",
+ "path": "groups/{uuid}/untrash",
+ "httpMethod": "POST",
+ "description": "untrash groups",
"parameters": {
"uuid": {
"type": "string",
"description": "The UUID of the KeepService in question.",
"required": true,
"location": "path"
+ },
+ "select": {
+ "type": "array",
+ "description": "Attributes of the object to return in the response.",
+ "required": false,
+ "location": "query"
}
},
"parameterOrder": [
"https://api.arvados.org/auth/arvados.readonly"
]
},
- "index": {
+ "list": {
"id": "arvados.keep_services.list",
"path": "keep_services",
"httpMethod": "GET",
"id": "arvados.keep_services.accessible",
"path": "keep_services/accessible",
"httpMethod": "GET",
- "description": "accessible keep_services",
- "parameters": {},
- "response": {
- "$ref": "KeepService"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados"
- ]
- },
- "list": {
- "id": "arvados.keep_services.list",
- "path": "keep_services",
- "httpMethod": "GET",
- "description": "List KeepServices.\n\n The <code>list</code> method returns a\n <a href=\"/api/resources.html\">resource list</a> of\n matching KeepServices. For example:\n\n <pre>\n {\n \"kind\":\"arvados#keepServiceList\",\n \"etag\":\"\",\n \"self_link\":\"\",\n \"next_page_token\":\"\",\n \"next_link\":\"\",\n \"items\":[\n ...\n ],\n \"items_available\":745,\n \"_profile\":{\n \"request_time\":0.157236317\n }\n </pre>",
- "parameters": {
- "filters": {
- "type": "array",
- "required": false,
- "description": "",
- "location": "query"
- },
- "where": {
- "type": "object",
- "required": false,
- "description": "",
- "location": "query"
- },
- "order": {
- "type": "array",
- "required": false,
- "description": "",
- "location": "query"
- },
- "select": {
- "type": "array",
- "description": "Attributes of each object to return in the response.",
- "required": false,
- "location": "query"
- },
- "distinct": {
- "type": "boolean",
- "required": false,
- "default": "false",
- "description": "",
- "location": "query"
- },
- "limit": {
- "type": "integer",
- "required": false,
- "default": "100",
- "description": "",
- "location": "query"
- },
- "offset": {
- "type": "integer",
- "required": false,
- "default": "0",
- "description": "",
- "location": "query"
- },
- "count": {
- "type": "string",
- "required": false,
- "default": "exact",
- "description": "",
- "location": "query"
- },
- "cluster_id": {
- "type": "string",
- "description": "List objects on a remote federated cluster instead of the current one.",
- "location": "query",
- "required": false
- },
- "bypass_federation": {
- "type": "boolean",
- "required": false,
- "description": "bypass federation behavior, list items from local instance database only",
- "location": "query"
- }
- },
- "response": {
- "$ref": "KeepServiceList"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados",
- "https://api.arvados.org/auth/arvados.readonly"
- ]
- },
- "show": {
- "id": "arvados.keep_services.show",
- "path": "keep_services/{uuid}",
- "httpMethod": "GET",
- "description": "show keep_services",
- "parameters": {
- "uuid": {
- "type": "string",
- "description": "",
- "required": true,
- "location": "path"
- },
- "select": {
- "type": "array",
- "description": "Attributes of the object to return in the response.",
- "required": false,
- "location": "query"
- }
- },
- "response": {
- "$ref": "KeepService"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados"
- ]
- },
- "destroy": {
- "id": "arvados.keep_services.destroy",
- "path": "keep_services/{uuid}",
- "httpMethod": "DELETE",
- "description": "destroy keep_services",
- "parameters": {
- "uuid": {
- "type": "string",
- "description": "",
- "required": true,
- "location": "path"
- }
- },
+ "description": "accessible keep_services",
+ "parameters": {},
"response": {
"$ref": "KeepService"
},
"description": "The UUID of the Link in question.",
"required": true,
"location": "path"
+ },
+ "select": {
+ "type": "array",
+ "description": "Attributes of the object to return in the response.",
+ "required": false,
+ "location": "query"
}
},
"parameterOrder": [
"https://api.arvados.org/auth/arvados.readonly"
]
},
- "index": {
+ "list": {
"id": "arvados.links.list",
"path": "links",
"httpMethod": "GET",
"https://api.arvados.org/auth/arvados"
]
},
- "list": {
- "id": "arvados.links.list",
- "path": "links",
- "httpMethod": "GET",
- "description": "List Links.\n\n The <code>list</code> method returns a\n <a href=\"/api/resources.html\">resource list</a> of\n matching Links. For example:\n\n <pre>\n {\n \"kind\":\"arvados#linkList\",\n \"etag\":\"\",\n \"self_link\":\"\",\n \"next_page_token\":\"\",\n \"next_link\":\"\",\n \"items\":[\n ...\n ],\n \"items_available\":745,\n \"_profile\":{\n \"request_time\":0.157236317\n }\n </pre>",
- "parameters": {
- "filters": {
- "type": "array",
- "required": false,
- "description": "",
- "location": "query"
- },
- "where": {
- "type": "object",
- "required": false,
- "description": "",
- "location": "query"
- },
- "order": {
- "type": "array",
- "required": false,
- "description": "",
- "location": "query"
- },
- "select": {
- "type": "array",
- "description": "Attributes of each object to return in the response.",
- "required": false,
- "location": "query"
- },
- "distinct": {
- "type": "boolean",
- "required": false,
- "default": "false",
- "description": "",
- "location": "query"
- },
- "limit": {
- "type": "integer",
- "required": false,
- "default": "100",
- "description": "",
- "location": "query"
- },
- "offset": {
- "type": "integer",
- "required": false,
- "default": "0",
- "description": "",
- "location": "query"
- },
- "count": {
- "type": "string",
- "required": false,
- "default": "exact",
- "description": "",
- "location": "query"
- },
- "cluster_id": {
- "type": "string",
- "description": "List objects on a remote federated cluster instead of the current one.",
- "location": "query",
- "required": false
- },
- "bypass_federation": {
- "type": "boolean",
- "required": false,
- "description": "bypass federation behavior, list items from local instance database only",
- "location": "query"
- }
- },
- "response": {
- "$ref": "LinkList"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados",
- "https://api.arvados.org/auth/arvados.readonly"
- ]
- },
- "show": {
- "id": "arvados.links.show",
- "path": "links/{uuid}",
- "httpMethod": "GET",
- "description": "show links",
- "parameters": {
- "uuid": {
- "type": "string",
- "description": "",
- "required": true,
- "location": "path"
- },
- "select": {
- "type": "array",
- "description": "Attributes of the object to return in the response.",
- "required": false,
- "location": "query"
- }
- },
- "response": {
- "$ref": "Link"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados"
- ]
- },
- "destroy": {
- "id": "arvados.links.destroy",
- "path": "links/{uuid}",
- "httpMethod": "DELETE",
- "description": "destroy links",
- "parameters": {
- "uuid": {
- "type": "string",
- "description": "",
- "required": true,
- "location": "path"
- }
- },
- "response": {
- "$ref": "Link"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados"
- ]
- },
"get_permissions": {
"id": "arvados.links.get_permissions",
"path": "permissions/{uuid}",
"description": "The UUID of the Log in question.",
"required": true,
"location": "path"
+ },
+ "select": {
+ "type": "array",
+ "description": "Attributes of the object to return in the response.",
+ "required": false,
+ "location": "query"
}
},
"parameterOrder": [
"https://api.arvados.org/auth/arvados.readonly"
]
},
- "index": {
+ "list": {
"id": "arvados.logs.list",
"path": "logs",
"httpMethod": "GET",
]
},
"delete": {
- "id": "arvados.logs.delete",
- "path": "logs/{uuid}",
- "httpMethod": "DELETE",
- "description": "Delete an existing Log.",
- "parameters": {
- "uuid": {
- "type": "string",
- "description": "The UUID of the Log in question.",
- "required": true,
- "location": "path"
- }
- },
- "response": {
- "$ref": "Log"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados"
- ]
- },
- "list": {
- "id": "arvados.logs.list",
- "path": "logs",
- "httpMethod": "GET",
- "description": "List Logs.\n\n The <code>list</code> method returns a\n <a href=\"/api/resources.html\">resource list</a> of\n matching Logs. For example:\n\n <pre>\n {\n \"kind\":\"arvados#logList\",\n \"etag\":\"\",\n \"self_link\":\"\",\n \"next_page_token\":\"\",\n \"next_link\":\"\",\n \"items\":[\n ...\n ],\n \"items_available\":745,\n \"_profile\":{\n \"request_time\":0.157236317\n }\n </pre>",
- "parameters": {
- "filters": {
- "type": "array",
- "required": false,
- "description": "",
- "location": "query"
- },
- "where": {
- "type": "object",
- "required": false,
- "description": "",
- "location": "query"
- },
- "order": {
- "type": "array",
- "required": false,
- "description": "",
- "location": "query"
- },
- "select": {
- "type": "array",
- "description": "Attributes of each object to return in the response.",
- "required": false,
- "location": "query"
- },
- "distinct": {
- "type": "boolean",
- "required": false,
- "default": "false",
- "description": "",
- "location": "query"
- },
- "limit": {
- "type": "integer",
- "required": false,
- "default": "100",
- "description": "",
- "location": "query"
- },
- "offset": {
- "type": "integer",
- "required": false,
- "default": "0",
- "description": "",
- "location": "query"
- },
- "count": {
- "type": "string",
- "required": false,
- "default": "exact",
- "description": "",
- "location": "query"
- },
- "cluster_id": {
- "type": "string",
- "description": "List objects on a remote federated cluster instead of the current one.",
- "location": "query",
- "required": false
- },
- "bypass_federation": {
- "type": "boolean",
- "required": false,
- "description": "bypass federation behavior, list items from local instance database only",
- "location": "query"
- }
- },
- "response": {
- "$ref": "LogList"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados",
- "https://api.arvados.org/auth/arvados.readonly"
- ]
- },
- "show": {
- "id": "arvados.logs.show",
- "path": "logs/{uuid}",
- "httpMethod": "GET",
- "description": "show logs",
- "parameters": {
- "uuid": {
- "type": "string",
- "description": "",
- "required": true,
- "location": "path"
- },
- "select": {
- "type": "array",
- "description": "Attributes of the object to return in the response.",
- "required": false,
- "location": "query"
- }
- },
- "response": {
- "$ref": "Log"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados"
- ]
- },
- "destroy": {
- "id": "arvados.logs.destroy",
+ "id": "arvados.logs.delete",
"path": "logs/{uuid}",
"httpMethod": "DELETE",
- "description": "destroy logs",
+ "description": "Delete an existing Log.",
"parameters": {
"uuid": {
"type": "string",
- "description": "",
+ "description": "The UUID of the Log in question.",
"required": true,
"location": "path"
}
"description": "The UUID of the User in question.",
"required": true,
"location": "path"
+ },
+ "select": {
+ "type": "array",
+ "description": "Attributes of the object to return in the response.",
+ "required": false,
+ "location": "query"
}
},
"parameterOrder": [
"https://api.arvados.org/auth/arvados.readonly"
]
},
- "index": {
+ "list": {
"id": "arvados.users.list",
"path": "users",
"httpMethod": "GET",
"scopes": [
"https://api.arvados.org/auth/arvados"
]
- },
- "list": {
- "id": "arvados.users.list",
- "path": "users",
- "httpMethod": "GET",
- "description": "List Users.\n\n The <code>list</code> method returns a\n <a href=\"/api/resources.html\">resource list</a> of\n matching Users. For example:\n\n <pre>\n {\n \"kind\":\"arvados#userList\",\n \"etag\":\"\",\n \"self_link\":\"\",\n \"next_page_token\":\"\",\n \"next_link\":\"\",\n \"items\":[\n ...\n ],\n \"items_available\":745,\n \"_profile\":{\n \"request_time\":0.157236317\n }\n </pre>",
- "parameters": {
- "filters": {
- "type": "array",
- "required": false,
- "description": "",
- "location": "query"
- },
- "where": {
- "type": "object",
- "required": false,
- "description": "",
- "location": "query"
- },
- "order": {
- "type": "array",
- "required": false,
- "description": "",
- "location": "query"
- },
- "select": {
- "type": "array",
- "description": "Attributes of each object to return in the response.",
- "required": false,
- "location": "query"
- },
- "distinct": {
- "type": "boolean",
- "required": false,
- "default": "false",
- "description": "",
- "location": "query"
- },
- "limit": {
- "type": "integer",
- "required": false,
- "default": "100",
- "description": "",
- "location": "query"
- },
- "offset": {
- "type": "integer",
- "required": false,
- "default": "0",
- "description": "",
- "location": "query"
- },
- "count": {
- "type": "string",
- "required": false,
- "default": "exact",
- "description": "",
- "location": "query"
- },
- "cluster_id": {
- "type": "string",
- "description": "List objects on a remote federated cluster instead of the current one.",
- "location": "query",
- "required": false
- },
- "bypass_federation": {
- "type": "boolean",
- "required": false,
- "description": "bypass federation behavior, list items from local instance database only",
- "location": "query"
- }
- },
- "response": {
- "$ref": "UserList"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados",
- "https://api.arvados.org/auth/arvados.readonly"
- ]
- },
- "show": {
- "id": "arvados.users.show",
- "path": "users/{uuid}",
- "httpMethod": "GET",
- "description": "show users",
- "parameters": {
- "uuid": {
- "type": "string",
- "description": "",
- "required": true,
- "location": "path"
- },
- "select": {
- "type": "array",
- "description": "Attributes of the object to return in the response.",
- "required": false,
- "location": "query"
- }
- },
- "response": {
- "$ref": "User"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados"
- ]
- },
- "destroy": {
- "id": "arvados.users.destroy",
- "path": "users/{uuid}",
- "httpMethod": "DELETE",
- "description": "destroy users",
- "parameters": {
- "uuid": {
- "type": "string",
- "description": "",
- "required": true,
- "location": "path"
- }
- },
- "response": {
- "$ref": "User"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados"
- ]
}
}
},
"description": "The UUID of the UserAgreement in question.",
"required": true,
"location": "path"
+ },
+ "select": {
+ "type": "array",
+ "description": "Attributes of the object to return in the response.",
+ "required": false,
+ "location": "query"
}
},
"parameterOrder": [
"https://api.arvados.org/auth/arvados.readonly"
]
},
- "index": {
+ "list": {
"id": "arvados.user_agreements.list",
"path": "user_agreements",
"httpMethod": "GET",
"https://api.arvados.org/auth/arvados"
]
},
- "list": {
- "id": "arvados.user_agreements.list",
- "path": "user_agreements",
- "httpMethod": "GET",
- "description": "List UserAgreements.\n\n The <code>list</code> method returns a\n <a href=\"/api/resources.html\">resource list</a> of\n matching UserAgreements. For example:\n\n <pre>\n {\n \"kind\":\"arvados#userAgreementList\",\n \"etag\":\"\",\n \"self_link\":\"\",\n \"next_page_token\":\"\",\n \"next_link\":\"\",\n \"items\":[\n ...\n ],\n \"items_available\":745,\n \"_profile\":{\n \"request_time\":0.157236317\n }\n </pre>",
- "parameters": {
- "filters": {
- "type": "array",
- "required": false,
- "description": "",
- "location": "query"
- },
- "where": {
- "type": "object",
- "required": false,
- "description": "",
- "location": "query"
- },
- "order": {
- "type": "array",
- "required": false,
- "description": "",
- "location": "query"
- },
- "select": {
- "type": "array",
- "description": "Attributes of each object to return in the response.",
- "required": false,
- "location": "query"
- },
- "distinct": {
- "type": "boolean",
- "required": false,
- "default": "false",
- "description": "",
- "location": "query"
- },
- "limit": {
- "type": "integer",
- "required": false,
- "default": "100",
- "description": "",
- "location": "query"
- },
- "offset": {
- "type": "integer",
- "required": false,
- "default": "0",
- "description": "",
- "location": "query"
- },
- "count": {
- "type": "string",
- "required": false,
- "default": "exact",
- "description": "",
- "location": "query"
- },
- "cluster_id": {
- "type": "string",
- "description": "List objects on a remote federated cluster instead of the current one.",
- "location": "query",
- "required": false
- },
- "bypass_federation": {
- "type": "boolean",
- "required": false,
- "description": "bypass federation behavior, list items from local instance database only",
- "location": "query"
- }
- },
- "response": {
- "$ref": "UserAgreementList"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados",
- "https://api.arvados.org/auth/arvados.readonly"
- ]
- },
"new": {
"id": "arvados.user_agreements.new",
"path": "user_agreements/new",
"scopes": [
"https://api.arvados.org/auth/arvados"
]
- },
- "show": {
- "id": "arvados.user_agreements.show",
- "path": "user_agreements/{uuid}",
- "httpMethod": "GET",
- "description": "show user_agreements",
- "parameters": {
- "uuid": {
- "type": "string",
- "description": "",
- "required": true,
- "location": "path"
- },
- "select": {
- "type": "array",
- "description": "Attributes of the object to return in the response.",
- "required": false,
- "location": "query"
- }
- },
- "response": {
- "$ref": "UserAgreement"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados"
- ]
- },
- "destroy": {
- "id": "arvados.user_agreements.destroy",
- "path": "user_agreements/{uuid}",
- "httpMethod": "DELETE",
- "description": "destroy user_agreements",
- "parameters": {
- "uuid": {
- "type": "string",
- "description": "",
- "required": true,
- "location": "path"
- }
- },
- "response": {
- "$ref": "UserAgreement"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados"
- ]
}
}
},
"description": "The UUID of the VirtualMachine in question.",
"required": true,
"location": "path"
+ },
+ "select": {
+ "type": "array",
+ "description": "Attributes of the object to return in the response.",
+ "required": false,
+ "location": "query"
}
},
"parameterOrder": [
"https://api.arvados.org/auth/arvados.readonly"
]
},
- "index": {
+ "list": {
"id": "arvados.virtual_machines.list",
"path": "virtual_machines",
"httpMethod": "GET",
"scopes": [
"https://api.arvados.org/auth/arvados"
]
- },
- "list": {
- "id": "arvados.virtual_machines.list",
- "path": "virtual_machines",
- "httpMethod": "GET",
- "description": "List VirtualMachines.\n\n The <code>list</code> method returns a\n <a href=\"/api/resources.html\">resource list</a> of\n matching VirtualMachines. For example:\n\n <pre>\n {\n \"kind\":\"arvados#virtualMachineList\",\n \"etag\":\"\",\n \"self_link\":\"\",\n \"next_page_token\":\"\",\n \"next_link\":\"\",\n \"items\":[\n ...\n ],\n \"items_available\":745,\n \"_profile\":{\n \"request_time\":0.157236317\n }\n </pre>",
- "parameters": {
- "filters": {
- "type": "array",
- "required": false,
- "description": "",
- "location": "query"
- },
- "where": {
- "type": "object",
- "required": false,
- "description": "",
- "location": "query"
- },
- "order": {
- "type": "array",
- "required": false,
- "description": "",
- "location": "query"
- },
- "select": {
- "type": "array",
- "description": "Attributes of each object to return in the response.",
- "required": false,
- "location": "query"
- },
- "distinct": {
- "type": "boolean",
- "required": false,
- "default": "false",
- "description": "",
- "location": "query"
- },
- "limit": {
- "type": "integer",
- "required": false,
- "default": "100",
- "description": "",
- "location": "query"
- },
- "offset": {
- "type": "integer",
- "required": false,
- "default": "0",
- "description": "",
- "location": "query"
- },
- "count": {
- "type": "string",
- "required": false,
- "default": "exact",
- "description": "",
- "location": "query"
- },
- "cluster_id": {
- "type": "string",
- "description": "List objects on a remote federated cluster instead of the current one.",
- "location": "query",
- "required": false
- },
- "bypass_federation": {
- "type": "boolean",
- "required": false,
- "description": "bypass federation behavior, list items from local instance database only",
- "location": "query"
- }
- },
- "response": {
- "$ref": "VirtualMachineList"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados",
- "https://api.arvados.org/auth/arvados.readonly"
- ]
- },
- "show": {
- "id": "arvados.virtual_machines.show",
- "path": "virtual_machines/{uuid}",
- "httpMethod": "GET",
- "description": "show virtual_machines",
- "parameters": {
- "uuid": {
- "type": "string",
- "description": "",
- "required": true,
- "location": "path"
- },
- "select": {
- "type": "array",
- "description": "Attributes of the object to return in the response.",
- "required": false,
- "location": "query"
- }
- },
- "response": {
- "$ref": "VirtualMachine"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados"
- ]
- },
- "destroy": {
- "id": "arvados.virtual_machines.destroy",
- "path": "virtual_machines/{uuid}",
- "httpMethod": "DELETE",
- "description": "destroy virtual_machines",
- "parameters": {
- "uuid": {
- "type": "string",
- "description": "",
- "required": true,
- "location": "path"
- }
- },
- "response": {
- "$ref": "VirtualMachine"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados"
- ]
}
}
},
"description": "The UUID of the Workflow in question.",
"required": true,
"location": "path"
+ },
+ "select": {
+ "type": "array",
+ "description": "Attributes of the object to return in the response.",
+ "required": false,
+ "location": "query"
}
},
"parameterOrder": [
"https://api.arvados.org/auth/arvados.readonly"
]
},
- "index": {
+ "list": {
"id": "arvados.workflows.list",
"path": "workflows",
"httpMethod": "GET",
"scopes": [
"https://api.arvados.org/auth/arvados"
]
- },
- "list": {
- "id": "arvados.workflows.list",
- "path": "workflows",
- "httpMethod": "GET",
- "description": "List Workflows.\n\n The <code>list</code> method returns a\n <a href=\"/api/resources.html\">resource list</a> of\n matching Workflows. For example:\n\n <pre>\n {\n \"kind\":\"arvados#workflowList\",\n \"etag\":\"\",\n \"self_link\":\"\",\n \"next_page_token\":\"\",\n \"next_link\":\"\",\n \"items\":[\n ...\n ],\n \"items_available\":745,\n \"_profile\":{\n \"request_time\":0.157236317\n }\n </pre>",
- "parameters": {
- "filters": {
- "type": "array",
- "required": false,
- "description": "",
- "location": "query"
- },
- "where": {
- "type": "object",
- "required": false,
- "description": "",
- "location": "query"
- },
- "order": {
- "type": "array",
- "required": false,
- "description": "",
- "location": "query"
- },
- "select": {
- "type": "array",
- "description": "Attributes of each object to return in the response.",
- "required": false,
- "location": "query"
- },
- "distinct": {
- "type": "boolean",
- "required": false,
- "default": "false",
- "description": "",
- "location": "query"
- },
- "limit": {
- "type": "integer",
- "required": false,
- "default": "100",
- "description": "",
- "location": "query"
- },
- "offset": {
- "type": "integer",
- "required": false,
- "default": "0",
- "description": "",
- "location": "query"
- },
- "count": {
- "type": "string",
- "required": false,
- "default": "exact",
- "description": "",
- "location": "query"
- },
- "cluster_id": {
- "type": "string",
- "description": "List objects on a remote federated cluster instead of the current one.",
- "location": "query",
- "required": false
- },
- "bypass_federation": {
- "type": "boolean",
- "required": false,
- "description": "bypass federation behavior, list items from local instance database only",
- "location": "query"
- }
- },
- "response": {
- "$ref": "WorkflowList"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados",
- "https://api.arvados.org/auth/arvados.readonly"
- ]
- },
- "show": {
- "id": "arvados.workflows.show",
- "path": "workflows/{uuid}",
- "httpMethod": "GET",
- "description": "show workflows",
- "parameters": {
- "uuid": {
- "type": "string",
- "description": "",
- "required": true,
- "location": "path"
- },
- "select": {
- "type": "array",
- "description": "Attributes of the object to return in the response.",
- "required": false,
- "location": "query"
- }
- },
- "response": {
- "$ref": "Workflow"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados"
- ]
- },
- "destroy": {
- "id": "arvados.workflows.destroy",
- "path": "workflows/{uuid}",
- "httpMethod": "DELETE",
- "description": "destroy workflows",
- "parameters": {
- "uuid": {
- "type": "string",
- "description": "",
- "required": true,
- "location": "path"
- }
- },
- "response": {
- "$ref": "Workflow"
- },
- "scopes": [
- "https://api.arvados.org/auth/arvados"
- ]
}
}
},
"api_client_id": {
"type": "integer"
},
- "user_id": {
- "type": "integer"
- },
"created_by_ip_address": {
"type": "string"
},
"created_at": {
"type": "datetime"
},
- "default_owner_uuid": {
- "type": "string"
- },
"scopes": {
"type": "Array"
}
from . import api, errors, util
from .api import api_from_config, http_cache
-from .collection import CollectionReader, CollectionWriter, ResumableCollectionWriter
+from .collection import CollectionReader
from arvados.keep import *
-from arvados.stream import *
from .arvfile import StreamFileReader
from .logging import log_format, log_date_format, log_handler
from .retry import RetryLoop
logger.addHandler(log_handler)
logger.setLevel(stdliblog.DEBUG if config.get('ARVADOS_DEBUG')
else stdliblog.WARNING)
-
-@util._deprecated('3.0', 'arvados-cwl-runner or the containers API')
-def task_set_output(self, s, num_retries=5):
- for tries_left in RetryLoop(num_retries=num_retries, backoff_start=0):
- try:
- return api('v1').job_tasks().update(
- uuid=self['uuid'],
- body={
- 'output':s,
- 'success':True,
- 'progress':1.0
- }).execute()
- except errors.ApiError as error:
- if retry.check_http_response_success(error.resp.status) is None and tries_left > 0:
- logger.debug("task_set_output: job_tasks().update() raised {}, retrying with {} tries left".format(repr(error),tries_left))
- else:
- raise
-
-_current_task = None
-@util._deprecated('3.0', 'arvados-cwl-runner or the containers API')
-def current_task(num_retries=5):
- global _current_task
- if _current_task:
- return _current_task
-
- for tries_left in RetryLoop(num_retries=num_retries, backoff_start=2):
- try:
- task = api('v1').job_tasks().get(uuid=os.environ['TASK_UUID']).execute()
- task = UserDict(task)
- task.set_output = types.MethodType(task_set_output, task)
- task.tmpdir = os.environ['TASK_WORK']
- _current_task = task
- return task
- except errors.ApiError as error:
- if retry.check_http_response_success(error.resp.status) is None and tries_left > 0:
- logger.debug("current_task: job_tasks().get() raised {}, retrying with {} tries left".format(repr(error),tries_left))
- else:
- raise
-
-_current_job = None
-@util._deprecated('3.0', 'arvados-cwl-runner or the containers API')
-def current_job(num_retries=5):
- global _current_job
- if _current_job:
- return _current_job
-
- for tries_left in RetryLoop(num_retries=num_retries, backoff_start=2):
- try:
- job = api('v1').jobs().get(uuid=os.environ['JOB_UUID']).execute()
- job = UserDict(job)
- job.tmpdir = os.environ['JOB_WORK']
- _current_job = job
- return job
- except errors.ApiError as error:
- if retry.check_http_response_success(error.resp.status) is None and tries_left > 0:
- logger.debug("current_job: jobs().get() raised {}, retrying with {} tries left".format(repr(error),tries_left))
- else:
- raise
-
-@util._deprecated('3.0', 'arvados-cwl-runner or the containers API')
-def getjobparam(*args):
- return current_job()['script_parameters'].get(*args)
-
-@util._deprecated('3.0', 'arvados-cwl-runner or the containers API')
-def get_job_param_mount(*args):
- return os.path.join(os.environ['TASK_KEEPMOUNT'], current_job()['script_parameters'].get(*args))
-
-@util._deprecated('3.0', 'arvados-cwl-runner or the containers API')
-def get_task_param_mount(*args):
- return os.path.join(os.environ['TASK_KEEPMOUNT'], current_task()['parameters'].get(*args))
-
-class JobTask(object):
- @util._deprecated('3.0', 'arvados-cwl-runner or the containers API')
- def __init__(self, parameters=dict(), runtime_constraints=dict()):
- print("init jobtask %s %s" % (parameters, runtime_constraints))
-
-class job_setup(object):
- @staticmethod
- @util._deprecated('3.0', 'arvados-cwl-runner or the containers API')
- def one_task_per_input_file(if_sequence=0, and_end_task=True, input_as_path=False, api_client=None):
- if if_sequence != current_task()['sequence']:
- return
-
- if not api_client:
- api_client = api('v1')
-
- job_input = current_job()['script_parameters']['input']
- cr = CollectionReader(job_input, api_client=api_client)
- cr.normalize()
- for s in cr.all_streams():
- for f in s.all_files():
- if input_as_path:
- task_input = os.path.join(job_input, s.name(), f.name())
- else:
- task_input = f.as_manifest()
- new_task_attrs = {
- 'job_uuid': current_job()['uuid'],
- 'created_by_job_task_uuid': current_task()['uuid'],
- 'sequence': if_sequence + 1,
- 'parameters': {
- 'input':task_input
- }
- }
- api_client.job_tasks().create(body=new_task_attrs).execute()
- if and_end_task:
- api_client.job_tasks().update(uuid=current_task()['uuid'],
- body={'success':True}
- ).execute()
- exit(0)
-
- @staticmethod
- @util._deprecated('3.0', 'arvados-cwl-runner or the containers API')
- def one_task_per_input_stream(if_sequence=0, and_end_task=True):
- if if_sequence != current_task()['sequence']:
- return
- job_input = current_job()['script_parameters']['input']
- cr = CollectionReader(job_input)
- for s in cr.all_streams():
- task_input = s.tokens()
- new_task_attrs = {
- 'job_uuid': current_job()['uuid'],
- 'created_by_job_task_uuid': current_task()['uuid'],
- 'sequence': if_sequence + 1,
- 'parameters': {
- 'input':task_input
- }
- }
- api('v1').job_tasks().create(body=new_task_attrs).execute()
- if and_end_task:
- api('v1').job_tasks().update(uuid=current_task()['uuid'],
- body={'success':True}
- ).execute()
- exit(0)
docstring for more information about their meaning.
"""
return api(**api_kwargs_from_config(version, apiconfig, **kwargs))
-
-class OrderedJsonModel(apiclient.model.JsonModel):
- """Model class for JSON that preserves the contents' order
-
- .. WARNING:: Deprecated
- This model is redundant now that Python dictionaries preserve insertion
- ordering. Code that passes this model to API constructors can remove it.
-
- In Python versions before 3.6, API clients that cared about preserving the
- order of fields in API server responses could use this model to do so.
- Typical usage looked like:
-
- from arvados.api import OrderedJsonModel
- client = arvados.api('v1', ..., model=OrderedJsonModel())
- """
- @util._deprecated(preferred="the default model and rely on Python's built-in dictionary ordering")
- def __init__(self, data_wrapper=False):
- return super().__init__(data_wrapper)
-
-
-RETRY_DELAY_INITIAL = 0
-"""
-.. WARNING:: Deprecated
- This constant was used by retry code in previous versions of the Arvados SDK.
- Changing the value has no effect anymore.
- Prefer passing `num_retries` to an API client constructor instead.
- Refer to the constructor docstrings for details.
-"""
-
-RETRY_DELAY_BACKOFF = 0
-"""
-.. WARNING:: Deprecated
- This constant was used by retry code in previous versions of the Arvados SDK.
- Changing the value has no effect anymore.
- Prefer passing `num_retries` to an API client constructor instead.
- Refer to the constructor docstrings for details.
-"""
-
-RETRY_COUNT = 0
-"""
-.. WARNING:: Deprecated
- This constant was used by retry code in previous versions of the Arvados SDK.
- Changing the value has no effect anymore.
- Prefer passing `num_retries` to an API client constructor instead.
- Refer to the constructor docstrings for details.
-"""
from .arvfile import split, _FileLikeObjectBase, ArvadosFile, ArvadosFileWriter, ArvadosFileReader, WrappableFile, _BlockManager, synchronized, must_be_writable, NoopLock
from .keep import KeepLocator, KeepClient
-from .stream import StreamReader
from ._normalize_stream import normalize_stream, escape
from ._ranges import Range, LocatorAndRange
from .safeapi import ThreadSafeApiCache
def writable(self) -> bool:
return self._in_init
-
- def _populate_streams(orig_func):
- @functools.wraps(orig_func)
- def populate_streams_wrapper(self, *args, **kwargs):
- # Defer populating self._streams until needed since it creates a copy of the manifest.
- if self._streams is None:
- if self._manifest_text:
- self._streams = [sline.split()
- for sline in self._manifest_text.split("\n")
- if sline]
- else:
- self._streams = []
- return orig_func(self, *args, **kwargs)
- return populate_streams_wrapper
-
- @arvados.util._deprecated('3.0', 'Collection iteration')
- @_populate_streams
- def normalize(self):
- """Normalize the streams returned by `all_streams`"""
- streams = {}
- for s in self.all_streams():
- for f in s.all_files():
- streamname, filename = split(s.name() + "/" + f.name())
- if streamname not in streams:
- streams[streamname] = {}
- if filename not in streams[streamname]:
- streams[streamname][filename] = []
- for r in f.segments:
- streams[streamname][filename].extend(s.locators_and_ranges(r.locator, r.range_size))
-
- self._streams = [normalize_stream(s, streams[s])
- for s in sorted(streams)]
-
- @arvados.util._deprecated('3.0', 'Collection iteration')
- @_populate_streams
- def all_streams(self):
- return [StreamReader(s, self._my_keep(), num_retries=self.num_retries)
- for s in self._streams]
-
- @arvados.util._deprecated('3.0', 'Collection iteration')
- @_populate_streams
- def all_files(self):
- for s in self.all_streams():
- for f in s.all_files():
- yield f
-
-
-class CollectionWriter(CollectionBase):
- """Create a new collection from scratch
-
- .. WARNING:: Deprecated
- This class is deprecated. Prefer `arvados.collection.Collection`
- instead.
- """
-
- @arvados.util._deprecated('3.0', 'arvados.collection.Collection')
- def __init__(self, api_client=None, num_retries=0, replication=None):
- """Instantiate a CollectionWriter.
-
- CollectionWriter lets you build a new Arvados Collection from scratch.
- Write files to it. The CollectionWriter will upload data to Keep as
- appropriate, and provide you with the Collection manifest text when
- you're finished.
-
- Arguments:
- * api_client: The API client to use to look up Collections. If not
- provided, CollectionReader will build one from available Arvados
- configuration.
- * num_retries: The default number of times to retry failed
- service requests. Default 0. You may change this value
- after instantiation, but note those changes may not
- propagate to related objects like the Keep client.
- * replication: The number of copies of each block to store.
- If this argument is None or not supplied, replication is
- the server-provided default if available, otherwise 2.
- """
- self._api_client = api_client
- self.num_retries = num_retries
- self.replication = (2 if replication is None else replication)
- self._keep_client = None
- self._data_buffer = []
- self._data_buffer_len = 0
- self._current_stream_files = []
- self._current_stream_length = 0
- self._current_stream_locators = []
- self._current_stream_name = '.'
- self._current_file_name = None
- self._current_file_pos = 0
- self._finished_streams = []
- self._close_file = None
- self._queued_file = None
- self._queued_dirents = deque()
- self._queued_trees = deque()
- self._last_open = None
-
- def __exit__(self, exc_type, exc_value, traceback):
- if exc_type is None:
- self.finish()
-
- def do_queued_work(self):
- # The work queue consists of three pieces:
- # * _queued_file: The file object we're currently writing to the
- # Collection.
- # * _queued_dirents: Entries under the current directory
- # (_queued_trees[0]) that we want to write or recurse through.
- # This may contain files from subdirectories if
- # max_manifest_depth == 0 for this directory.
- # * _queued_trees: Directories that should be written as separate
- # streams to the Collection.
- # This function handles the smallest piece of work currently queued
- # (current file, then current directory, then next directory) until
- # no work remains. The _work_THING methods each do a unit of work on
- # THING. _queue_THING methods add a THING to the work queue.
- while True:
- if self._queued_file:
- self._work_file()
- elif self._queued_dirents:
- self._work_dirents()
- elif self._queued_trees:
- self._work_trees()
- else:
- break
-
- def _work_file(self):
- while True:
- buf = self._queued_file.read(config.KEEP_BLOCK_SIZE)
- if not buf:
- break
- self.write(buf)
- self.finish_current_file()
- if self._close_file:
- self._queued_file.close()
- self._close_file = None
- self._queued_file = None
-
- def _work_dirents(self):
- path, stream_name, max_manifest_depth = self._queued_trees[0]
- if stream_name != self.current_stream_name():
- self.start_new_stream(stream_name)
- while self._queued_dirents:
- dirent = self._queued_dirents.popleft()
- target = os.path.join(path, dirent)
- if os.path.isdir(target):
- self._queue_tree(target,
- os.path.join(stream_name, dirent),
- max_manifest_depth - 1)
- else:
- self._queue_file(target, dirent)
- break
- if not self._queued_dirents:
- self._queued_trees.popleft()
-
- def _work_trees(self):
- path, stream_name, max_manifest_depth = self._queued_trees[0]
- d = arvados.util.listdir_recursive(
- path, max_depth = (None if max_manifest_depth == 0 else 0))
- if d:
- self._queue_dirents(stream_name, d)
- else:
- self._queued_trees.popleft()
-
- def _queue_file(self, source, filename=None):
- assert (self._queued_file is None), "tried to queue more than one file"
- if not hasattr(source, 'read'):
- source = open(source, 'rb')
- self._close_file = True
- else:
- self._close_file = False
- if filename is None:
- filename = os.path.basename(source.name)
- self.start_new_file(filename)
- self._queued_file = source
-
- def _queue_dirents(self, stream_name, dirents):
- assert (not self._queued_dirents), "tried to queue more than one tree"
- self._queued_dirents = deque(sorted(dirents))
-
- def _queue_tree(self, path, stream_name, max_manifest_depth):
- self._queued_trees.append((path, stream_name, max_manifest_depth))
-
- def write_file(self, source, filename=None):
- self._queue_file(source, filename)
- self.do_queued_work()
-
- def write_directory_tree(self,
- path, stream_name='.', max_manifest_depth=-1):
- self._queue_tree(path, stream_name, max_manifest_depth)
- self.do_queued_work()
-
- def write(self, newdata):
- if isinstance(newdata, bytes):
- pass
- elif isinstance(newdata, str):
- newdata = newdata.encode()
- elif hasattr(newdata, '__iter__'):
- for s in newdata:
- self.write(s)
- return
- self._data_buffer.append(newdata)
- self._data_buffer_len += len(newdata)
- self._current_stream_length += len(newdata)
- while self._data_buffer_len >= config.KEEP_BLOCK_SIZE:
- self.flush_data()
-
- def open(self, streampath, filename=None):
- """open(streampath[, filename]) -> file-like object
-
- Pass in the path of a file to write to the Collection, either as a
- single string or as two separate stream name and file name arguments.
- This method returns a file-like object you can write to add it to the
- Collection.
-
- You may only have one file object from the Collection open at a time,
- so be sure to close the object when you're done. Using the object in
- a with statement makes that easy:
-
- with cwriter.open('./doc/page1.txt') as outfile:
- outfile.write(page1_data)
- with cwriter.open('./doc/page2.txt') as outfile:
- outfile.write(page2_data)
- """
- if filename is None:
- streampath, filename = split(streampath)
- if self._last_open and not self._last_open.closed:
- raise errors.AssertionError(
- u"can't open '{}' when '{}' is still open".format(
- filename, self._last_open.name))
- if streampath != self.current_stream_name():
- self.start_new_stream(streampath)
- self.set_current_file_name(filename)
- self._last_open = _WriterFile(self, filename)
- return self._last_open
-
- def flush_data(self):
- data_buffer = b''.join(self._data_buffer)
- if data_buffer:
- self._current_stream_locators.append(
- self._my_keep().put(
- data_buffer[0:config.KEEP_BLOCK_SIZE],
- copies=self.replication))
- self._data_buffer = [data_buffer[config.KEEP_BLOCK_SIZE:]]
- self._data_buffer_len = len(self._data_buffer[0])
-
- def start_new_file(self, newfilename=None):
- self.finish_current_file()
- self.set_current_file_name(newfilename)
-
- def set_current_file_name(self, newfilename):
- if re.search(r'[\t\n]', newfilename):
- raise errors.AssertionError(
- "Manifest filenames cannot contain whitespace: %s" %
- newfilename)
- elif re.search(r'\x00', newfilename):
- raise errors.AssertionError(
- "Manifest filenames cannot contain NUL characters: %s" %
- newfilename)
- self._current_file_name = newfilename
-
- def current_file_name(self):
- return self._current_file_name
-
- def finish_current_file(self):
- if self._current_file_name is None:
- if self._current_file_pos == self._current_stream_length:
- return
- raise errors.AssertionError(
- "Cannot finish an unnamed file " +
- "(%d bytes at offset %d in '%s' stream)" %
- (self._current_stream_length - self._current_file_pos,
- self._current_file_pos,
- self._current_stream_name))
- self._current_stream_files.append([
- self._current_file_pos,
- self._current_stream_length - self._current_file_pos,
- self._current_file_name])
- self._current_file_pos = self._current_stream_length
- self._current_file_name = None
-
- def start_new_stream(self, newstreamname='.'):
- self.finish_current_stream()
- self.set_current_stream_name(newstreamname)
-
- def set_current_stream_name(self, newstreamname):
- if re.search(r'[\t\n]', newstreamname):
- raise errors.AssertionError(
- "Manifest stream names cannot contain whitespace: '%s'" %
- (newstreamname))
- self._current_stream_name = '.' if newstreamname=='' else newstreamname
-
- def current_stream_name(self):
- return self._current_stream_name
-
- def finish_current_stream(self):
- self.finish_current_file()
- self.flush_data()
- if not self._current_stream_files:
- pass
- elif self._current_stream_name is None:
- raise errors.AssertionError(
- "Cannot finish an unnamed stream (%d bytes in %d files)" %
- (self._current_stream_length, len(self._current_stream_files)))
- else:
- if not self._current_stream_locators:
- self._current_stream_locators.append(config.EMPTY_BLOCK_LOCATOR)
- self._finished_streams.append([self._current_stream_name,
- self._current_stream_locators,
- self._current_stream_files])
- self._current_stream_files = []
- self._current_stream_length = 0
- self._current_stream_locators = []
- self._current_stream_name = None
- self._current_file_pos = 0
- self._current_file_name = None
-
- def finish(self):
- """Store the manifest in Keep and return its locator.
-
- This is useful for storing manifest fragments (task outputs)
- temporarily in Keep during a Crunch job.
-
- In other cases you should make a collection instead, by
- sending manifest_text() to the API server's "create
- collection" endpoint.
- """
- return self._my_keep().put(self.manifest_text().encode(),
- copies=self.replication)
-
- def portable_data_hash(self):
- stripped = self.stripped_manifest().encode()
- return '{}+{}'.format(hashlib.md5(stripped).hexdigest(), len(stripped))
-
- def manifest_text(self):
- self.finish_current_stream()
- manifest = ''
-
- for stream in self._finished_streams:
- if not re.search(r'^\.(/.*)?$', stream[0]):
- manifest += './'
- manifest += stream[0].replace(' ', '\\040')
- manifest += ' ' + ' '.join(stream[1])
- manifest += ' ' + ' '.join("%d:%d:%s" % (sfile[0], sfile[1], sfile[2].replace(' ', '\\040')) for sfile in stream[2])
- manifest += "\n"
-
- return manifest
-
- def data_locators(self):
- ret = []
- for name, locators, files in self._finished_streams:
- ret += locators
- return ret
-
- def save_new(self, name=None):
- return self._api_client.collections().create(
- ensure_unique_name=True,
- body={
- 'name': name,
- 'manifest_text': self.manifest_text(),
- }).execute(num_retries=self.num_retries)
-
-
-class ResumableCollectionWriter(CollectionWriter):
- """CollectionWriter that can serialize internal state to disk
-
- .. WARNING:: Deprecated
- This class is deprecated. Prefer `arvados.collection.Collection`
- instead.
- """
-
- STATE_PROPS = ['_current_stream_files', '_current_stream_length',
- '_current_stream_locators', '_current_stream_name',
- '_current_file_name', '_current_file_pos', '_close_file',
- '_data_buffer', '_dependencies', '_finished_streams',
- '_queued_dirents', '_queued_trees']
-
- @arvados.util._deprecated('3.0', 'arvados.collection.Collection')
- def __init__(self, api_client=None, **kwargs):
- self._dependencies = {}
- super(ResumableCollectionWriter, self).__init__(api_client, **kwargs)
-
- @classmethod
- def from_state(cls, state, *init_args, **init_kwargs):
- # Try to build a new writer from scratch with the given state.
- # If the state is not suitable to resume (because files have changed,
- # been deleted, aren't predictable, etc.), raise a
- # StaleWriterStateError. Otherwise, return the initialized writer.
- # The caller is responsible for calling writer.do_queued_work()
- # appropriately after it's returned.
- writer = cls(*init_args, **init_kwargs)
- for attr_name in cls.STATE_PROPS:
- attr_value = state[attr_name]
- attr_class = getattr(writer, attr_name).__class__
- # Coerce the value into the same type as the initial value, if
- # needed.
- if attr_class not in (type(None), attr_value.__class__):
- attr_value = attr_class(attr_value)
- setattr(writer, attr_name, attr_value)
- # Check dependencies before we try to resume anything.
- if any(KeepLocator(ls).permission_expired()
- for ls in writer._current_stream_locators):
- raise errors.StaleWriterStateError(
- "locators include expired permission hint")
- writer.check_dependencies()
- if state['_current_file'] is not None:
- path, pos = state['_current_file']
- try:
- writer._queued_file = open(path, 'rb')
- writer._queued_file.seek(pos)
- except IOError as error:
- raise errors.StaleWriterStateError(
- u"failed to reopen active file {}: {}".format(path, error))
- return writer
-
- def check_dependencies(self):
- for path, orig_stat in self._dependencies.items():
- if not S_ISREG(orig_stat[ST_MODE]):
- raise errors.StaleWriterStateError(u"{} not file".format(path))
- try:
- now_stat = tuple(os.stat(path))
- except OSError as error:
- raise errors.StaleWriterStateError(
- u"failed to stat {}: {}".format(path, error))
- if ((not S_ISREG(now_stat[ST_MODE])) or
- (orig_stat[ST_MTIME] != now_stat[ST_MTIME]) or
- (orig_stat[ST_SIZE] != now_stat[ST_SIZE])):
- raise errors.StaleWriterStateError(u"{} changed".format(path))
-
- def dump_state(self, copy_func=lambda x: x):
- state = {attr: copy_func(getattr(self, attr))
- for attr in self.STATE_PROPS}
- if self._queued_file is None:
- state['_current_file'] = None
- else:
- state['_current_file'] = (os.path.realpath(self._queued_file.name),
- self._queued_file.tell())
- return state
-
- def _queue_file(self, source, filename=None):
- try:
- src_path = os.path.realpath(source)
- except Exception:
- raise errors.AssertionError(u"{} not a file path".format(source))
- try:
- path_stat = os.stat(src_path)
- except OSError as stat_error:
- path_stat = None
- super(ResumableCollectionWriter, self)._queue_file(source, filename)
- fd_stat = os.fstat(self._queued_file.fileno())
- if not S_ISREG(fd_stat.st_mode):
- # We won't be able to resume from this cache anyway, so don't
- # worry about further checks.
- self._dependencies[source] = tuple(fd_stat)
- elif path_stat is None:
- raise errors.AssertionError(
- u"could not stat {}: {}".format(source, stat_error))
- elif path_stat.st_ino != fd_stat.st_ino:
- raise errors.AssertionError(
- u"{} changed between open and stat calls".format(source))
- else:
- self._dependencies[src_path] = tuple(fd_stat)
-
- def write(self, data):
- if self._queued_file is None:
- raise errors.AssertionError(
- "resumable writer can't accept unsourced data")
- return super(ResumableCollectionWriter, self).write(data)
key = (img["repo"], img["tag"], img["timestamp"])
old_images.append(img)
- migration_links = arvados.util.list_all(api_client.links().list, filters=[
- ['link_class', '=', _migration_link_class],
- ['name', '=', _migration_link_name],
- ])
+ migration_links = arvados.util.keyset_list_all(
+ api_client.links().list,
+ filters=[
+ ['link_class', '=', _migration_link_class],
+ ['name', '=', _migration_link_name],
+ ],
+ order='uuid')
already_migrated = set()
for m in migration_links:
already_migrated.add(m["tail_uuid"])
- items = arvados.util.list_all(api_client.collections().list,
- filters=[["uuid", "in", [img["collection"] for img in old_images]]],
- select=["uuid", "portable_data_hash", "manifest_text", "owner_uuid"])
+ items = arvados.util.keyset_list_all(
+ api_client.collections().list,
+ filters=[["uuid", "in", [img["collection"] for img in old_images]]],
+ select=["uuid", "portable_data_hash", "manifest_text", "owner_uuid"],
+ order='uuid')
uuid_to_collection = {i["uuid"]: i for i in items}
need_migrate = {}
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-
-import json
-import os
-
-from . import util
-
-class TaskOutputDir(object):
- """Keep-backed directory for staging outputs of Crunch tasks.
-
- Example, in a crunch task whose output is a file called "out.txt"
- containing "42":
-
- import arvados
- import arvados.crunch
- import os
-
- out = arvados.crunch.TaskOutputDir()
- with open(os.path.join(out.path, 'out.txt'), 'w') as f:
- f.write('42')
- arvados.current_task().set_output(out.manifest_text())
- """
- @util._deprecated('3.0', 'arvados-cwl-runner or the containers API')
- def __init__(self):
- self.path = os.environ['TASK_KEEPMOUNT_TMP']
-
- def __str__(self):
- return self.path
-
- def manifest_text(self):
- snapshot = os.path.join(self.path, '.arvados#collection')
- return json.load(open(snapshot))['manifest_text']
This method runs in a separate thread to receive and process events
from the server.
"""
- self.setName(f'ArvadosWebsockets-{self.ident}')
+ self.name = f'ArvadosWebsockets-{self.ident}'
while self._client_ok and not self.is_closed.is_set():
try:
with self._subscribe_lock:
return self.perm_expiry <= as_of_dt
-class Keep(object):
- """Simple interface to a global KeepClient object.
-
- THIS CLASS IS DEPRECATED. Please instantiate your own KeepClient with your
- own API client. The global KeepClient will build an API client from the
- current Arvados configuration, which may not match the one you built.
- """
- _last_key = None
-
- @classmethod
- def global_client_object(cls):
- global global_client_object
- # Previously, KeepClient would change its behavior at runtime based
- # on these configuration settings. We simulate that behavior here
- # by checking the values and returning a new KeepClient if any of
- # them have changed.
- key = (config.get('ARVADOS_API_HOST'),
- config.get('ARVADOS_API_TOKEN'),
- config.flag_is_true('ARVADOS_API_HOST_INSECURE'),
- config.get('ARVADOS_KEEP_PROXY'),
- os.environ.get('KEEP_LOCAL_STORE'))
- if (global_client_object is None) or (cls._last_key != key):
- global_client_object = KeepClient()
- cls._last_key = key
- return global_client_object
-
- @staticmethod
- def get(locator, **kwargs):
- return Keep.global_client_object().get(locator, **kwargs)
-
- @staticmethod
- def put(data, **kwargs):
- return Keep.global_client_object().put(data, **kwargs)
-
class KeepBlockCache(object):
def __init__(self, cache_max=0, max_slots=0, disk_cache=False, disk_cache_dir=None):
self.cache_max = cache_max
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-
-import collections
-import hashlib
-import os
-import re
-import threading
-import functools
-import copy
-
-from ._ranges import locators_and_ranges, Range
-from .arvfile import StreamFileReader
-from arvados.retry import retry_method
-from arvados.keep import *
-from . import config
-from . import errors
-from . import util
-from ._normalize_stream import normalize_stream
-
-class StreamReader(object):
- @util._deprecated('3.0', 'arvados.collection.Collecttion')
- def __init__(self, tokens, keep=None, debug=False, _empty=False,
- num_retries=10):
- self._stream_name = None
- self._data_locators = []
- self._files = collections.OrderedDict()
- self._keep = keep
- self.num_retries = num_retries
-
- streamoffset = 0
-
- # parse stream
- for tok in tokens:
- if debug: print('tok', tok)
- if self._stream_name is None:
- self._stream_name = tok.replace('\\040', ' ')
- continue
-
- s = re.match(r'^[0-9a-f]{32}\+(\d+)(\+\S+)*$', tok)
- if s:
- blocksize = int(s.group(1))
- self._data_locators.append(Range(tok, streamoffset, blocksize, 0))
- streamoffset += blocksize
- continue
-
- s = re.search(r'^(\d+):(\d+):(\S+)', tok)
- if s:
- pos = int(s.group(1))
- size = int(s.group(2))
- name = s.group(3).replace('\\040', ' ')
- if name not in self._files:
- self._files[name] = StreamFileReader(self, [Range(pos, 0, size, 0)], name)
- else:
- filereader = self._files[name]
- filereader.segments.append(Range(pos, filereader.size(), size))
- continue
-
- raise errors.SyntaxError("Invalid manifest format")
-
- def name(self):
- return self._stream_name
-
- def files(self):
- return self._files
-
- def all_files(self):
- return list(self._files.values())
-
- def size(self):
- n = self._data_locators[-1]
- return n.range_start + n.range_size
-
- def locators_and_ranges(self, range_start, range_size):
- return locators_and_ranges(self._data_locators, range_start, range_size)
-
- @retry_method
- def _keepget(self, locator, num_retries=None):
- return self._keep.get(locator, num_retries=num_retries)
-
- @retry_method
- def readfrom(self, start, size, num_retries=None):
- """Read up to 'size' bytes from the stream, starting at 'start'"""
- if size == 0:
- return b''
- if self._keep is None:
- self._keep = KeepClient(num_retries=self.num_retries)
- data = []
- for lr in locators_and_ranges(self._data_locators, start, size):
- data.append(self._keepget(lr.locator, num_retries=num_retries)[lr.segment_offset:lr.segment_offset+lr.segment_size])
- return b''.join(data)
-
- def manifest_text(self, strip=False):
- manifest_text = [self.name().replace(' ', '\\040')]
- if strip:
- for d in self._data_locators:
- m = re.match(r'^[0-9a-f]{32}\+\d+', d.locator)
- manifest_text.append(m.group(0))
- else:
- manifest_text.extend([d.locator for d in self._data_locators])
- manifest_text.extend([' '.join(["{}:{}:{}".format(seg.locator, seg.range_size, f.name.replace(' ', '\\040'))
- for seg in f.segments])
- for f in self._files.values()])
- return ' '.join(manifest_text) + '\n'
"""Regular expression to match any Arvados link UUID"""
user_uuid_pattern = re.compile(r'[a-z0-9]{5}-tpzed-[a-z0-9]{15}')
"""Regular expression to match any Arvados user UUID"""
-job_uuid_pattern = re.compile(r'[a-z0-9]{5}-8i9sb-[a-z0-9]{15}')
-"""Regular expression to match any Arvados job UUID
-
-.. WARNING:: Deprecated
- Arvados job resources are deprecated and will be removed in a future
- release. Prefer the containers API instead.
-"""
logger = logging.getLogger('arvados')
collectionname = collectionname[0:split] + "…" + collectionname[split+over:]
return collectionname
-
-@_deprecated('3.0', 'arvados.util.keyset_list_all')
-def list_all(fn, num_retries=0, **kwargs):
- # Default limit to (effectively) api server's MAX_LIMIT
- kwargs.setdefault('limit', sys.maxsize)
- items = []
- offset = 0
- items_available = sys.maxsize
- while len(items) < items_available:
- c = fn(offset=offset, **kwargs).execute(num_retries=num_retries)
- items += c['items']
- items_available = c['items_available']
- offset = c['offset'] + len(c['items'])
- return items
-
-@_deprecated('3.0')
-def clear_tmpdir(path=None):
- """
- Ensure the given directory (or TASK_TMPDIR if none given)
- exists and is empty.
- """
- from arvados import current_task
- if path is None:
- path = current_task().tmpdir
- if os.path.exists(path):
- p = subprocess.Popen(['rm', '-rf', path])
- stdout, stderr = p.communicate(None)
- if p.returncode != 0:
- raise Exception('rm -rf %s: %s' % (path, stderr))
- os.mkdir(path)
-
-@_deprecated('3.0', 'subprocess.run')
-def run_command(execargs, **kwargs):
- kwargs.setdefault('stdin', subprocess.PIPE)
- kwargs.setdefault('stdout', subprocess.PIPE)
- kwargs.setdefault('stderr', sys.stderr)
- kwargs.setdefault('close_fds', True)
- kwargs.setdefault('shell', False)
- p = subprocess.Popen(execargs, **kwargs)
- stdoutdata, stderrdata = p.communicate(None)
- if p.returncode != 0:
- raise arvados.errors.CommandFailedError(
- "run_command %s exit %d:\n%s" %
- (execargs, p.returncode, stderrdata))
- return stdoutdata, stderrdata
-
-@_deprecated('3.0')
-def git_checkout(url, version, path):
- from arvados import current_job
- if not re.search('^/', path):
- path = os.path.join(current_job().tmpdir, path)
- if not os.path.exists(path):
- run_command(["git", "clone", url, path],
- cwd=os.path.dirname(path))
- run_command(["git", "checkout", version],
- cwd=path)
- return path
-
-@_deprecated('3.0')
-def tar_extractor(path, decompress_flag):
- return subprocess.Popen(["tar",
- "-C", path,
- ("-x%sf" % decompress_flag),
- "-"],
- stdout=None,
- stdin=subprocess.PIPE, stderr=sys.stderr,
- shell=False, close_fds=True)
-
-@_deprecated('3.0', 'arvados.collection.Collection.open and the tarfile module')
-def tarball_extract(tarball, path):
- """Retrieve a tarball from Keep and extract it to a local
- directory. Return the absolute path where the tarball was
- extracted. If the top level of the tarball contained just one
- file or directory, return the absolute path of that single
- item.
-
- tarball -- collection locator
- path -- where to extract the tarball: absolute, or relative to job tmp
- """
- from arvados import current_job
- from arvados.collection import CollectionReader
- if not re.search('^/', path):
- path = os.path.join(current_job().tmpdir, path)
- lockfile = open(path + '.lock', 'w')
- fcntl.flock(lockfile, fcntl.LOCK_EX)
- try:
- os.stat(path)
- except OSError:
- os.mkdir(path)
- already_have_it = False
- try:
- if os.readlink(os.path.join(path, '.locator')) == tarball:
- already_have_it = True
- except OSError:
- pass
- if not already_have_it:
-
- # emulate "rm -f" (i.e., if the file does not exist, we win)
- try:
- os.unlink(os.path.join(path, '.locator'))
- except OSError:
- if os.path.exists(os.path.join(path, '.locator')):
- os.unlink(os.path.join(path, '.locator'))
-
- for f in CollectionReader(tarball).all_files():
- f_name = f.name()
- if f_name.endswith(('.tbz', '.tar.bz2')):
- p = tar_extractor(path, 'j')
- elif f_name.endswith(('.tgz', '.tar.gz')):
- p = tar_extractor(path, 'z')
- elif f_name.endswith('.tar'):
- p = tar_extractor(path, '')
- else:
- raise arvados.errors.AssertionError(
- "tarball_extract cannot handle filename %s" % f.name())
- while True:
- buf = f.read(2**20)
- if len(buf) == 0:
- break
- p.stdin.write(buf)
- p.stdin.close()
- p.wait()
- if p.returncode != 0:
- lockfile.close()
- raise arvados.errors.CommandFailedError(
- "tar exited %d" % p.returncode)
- os.symlink(tarball, os.path.join(path, '.locator'))
- tld_extracts = [f for f in os.listdir(path) if f != '.locator']
- lockfile.close()
- if len(tld_extracts) == 1:
- return os.path.join(path, tld_extracts[0])
- return path
-
-@_deprecated('3.0', 'arvados.collection.Collection.open and the zipfile module')
-def zipball_extract(zipball, path):
- """Retrieve a zip archive from Keep and extract it to a local
- directory. Return the absolute path where the archive was
- extracted. If the top level of the archive contained just one
- file or directory, return the absolute path of that single
- item.
-
- zipball -- collection locator
- path -- where to extract the archive: absolute, or relative to job tmp
- """
- from arvados import current_job
- from arvados.collection import CollectionReader
- if not re.search('^/', path):
- path = os.path.join(current_job().tmpdir, path)
- lockfile = open(path + '.lock', 'w')
- fcntl.flock(lockfile, fcntl.LOCK_EX)
- try:
- os.stat(path)
- except OSError:
- os.mkdir(path)
- already_have_it = False
- try:
- if os.readlink(os.path.join(path, '.locator')) == zipball:
- already_have_it = True
- except OSError:
- pass
- if not already_have_it:
-
- # emulate "rm -f" (i.e., if the file does not exist, we win)
- try:
- os.unlink(os.path.join(path, '.locator'))
- except OSError:
- if os.path.exists(os.path.join(path, '.locator')):
- os.unlink(os.path.join(path, '.locator'))
-
- for f in CollectionReader(zipball).all_files():
- if not f.name().endswith('.zip'):
- raise arvados.errors.NotImplementedError(
- "zipball_extract cannot handle filename %s" % f.name())
- zip_filename = os.path.join(path, os.path.basename(f.name()))
- zip_file = open(zip_filename, 'wb')
- while True:
- buf = f.read(2**20)
- if len(buf) == 0:
- break
- zip_file.write(buf)
- zip_file.close()
-
- p = subprocess.Popen(["unzip",
- "-q", "-o",
- "-d", path,
- zip_filename],
- stdout=None,
- stdin=None, stderr=sys.stderr,
- shell=False, close_fds=True)
- p.wait()
- if p.returncode != 0:
- lockfile.close()
- raise arvados.errors.CommandFailedError(
- "unzip exited %d" % p.returncode)
- os.unlink(zip_filename)
- os.symlink(zipball, os.path.join(path, '.locator'))
- tld_extracts = [f for f in os.listdir(path) if f != '.locator']
- lockfile.close()
- if len(tld_extracts) == 1:
- return os.path.join(path, tld_extracts[0])
- return path
-
-@_deprecated('3.0', 'arvados.collection.Collection')
-def collection_extract(collection, path, files=[], decompress=True):
- """Retrieve a collection from Keep and extract it to a local
- directory. Return the absolute path where the collection was
- extracted.
-
- collection -- collection locator
- path -- where to extract: absolute, or relative to job tmp
- """
- from arvados import current_job
- from arvados.collection import CollectionReader
- matches = re.search(r'^([0-9a-f]+)(\+[\w@]+)*$', collection)
- if matches:
- collection_hash = matches.group(1)
- else:
- collection_hash = hashlib.md5(collection).hexdigest()
- if not re.search('^/', path):
- path = os.path.join(current_job().tmpdir, path)
- lockfile = open(path + '.lock', 'w')
- fcntl.flock(lockfile, fcntl.LOCK_EX)
- try:
- os.stat(path)
- except OSError:
- os.mkdir(path)
- already_have_it = False
- try:
- if os.readlink(os.path.join(path, '.locator')) == collection_hash:
- already_have_it = True
- except OSError:
- pass
-
- # emulate "rm -f" (i.e., if the file does not exist, we win)
- try:
- os.unlink(os.path.join(path, '.locator'))
- except OSError:
- if os.path.exists(os.path.join(path, '.locator')):
- os.unlink(os.path.join(path, '.locator'))
-
- files_got = []
- for s in CollectionReader(collection).all_streams():
- stream_name = s.name()
- for f in s.all_files():
- if (files == [] or
- ((f.name() not in files_got) and
- (f.name() in files or
- (decompress and f.decompressed_name() in files)))):
- outname = f.decompressed_name() if decompress else f.name()
- files_got += [outname]
- if os.path.exists(os.path.join(path, stream_name, outname)):
- continue
- mkdir_dash_p(os.path.dirname(os.path.join(path, stream_name, outname)))
- outfile = open(os.path.join(path, stream_name, outname), 'wb')
- for buf in (f.readall_decompressed() if decompress
- else f.readall()):
- outfile.write(buf)
- outfile.close()
- if len(files_got) < len(files):
- raise arvados.errors.AssertionError(
- "Wanted files %s but only got %s from %s" %
- (files, files_got,
- [z.name() for z in CollectionReader(collection).all_files()]))
- os.symlink(collection_hash, os.path.join(path, '.locator'))
-
- lockfile.close()
- return path
-
-@_deprecated('3.0', 'pathlib.Path().mkdir(parents=True, exist_ok=True)')
-def mkdir_dash_p(path):
- if not os.path.isdir(path):
- try:
- os.makedirs(path)
- except OSError as e:
- if e.errno == errno.EEXIST and os.path.isdir(path):
- # It is not an error if someone else creates the
- # directory between our exists() and makedirs() calls.
- pass
- else:
- raise
-
-@_deprecated('3.0', 'arvados.collection.Collection')
-def stream_extract(stream, path, files=[], decompress=True):
- """Retrieve a stream from Keep and extract it to a local
- directory. Return the absolute path where the stream was
- extracted.
-
- stream -- StreamReader object
- path -- where to extract: absolute, or relative to job tmp
- """
- from arvados import current_job
- if not re.search('^/', path):
- path = os.path.join(current_job().tmpdir, path)
- lockfile = open(path + '.lock', 'w')
- fcntl.flock(lockfile, fcntl.LOCK_EX)
- try:
- os.stat(path)
- except OSError:
- os.mkdir(path)
-
- files_got = []
- for f in stream.all_files():
- if (files == [] or
- ((f.name() not in files_got) and
- (f.name() in files or
- (decompress and f.decompressed_name() in files)))):
- outname = f.decompressed_name() if decompress else f.name()
- files_got += [outname]
- if os.path.exists(os.path.join(path, outname)):
- os.unlink(os.path.join(path, outname))
- mkdir_dash_p(os.path.dirname(os.path.join(path, outname)))
- outfile = open(os.path.join(path, outname), 'wb')
- for buf in (f.readall_decompressed() if decompress
- else f.readall()):
- outfile.write(buf)
- outfile.close()
- if len(files_got) < len(files):
- raise arvados.errors.AssertionError(
- "Wanted files %s but only got %s from %s" %
- (files, files_got, [z.name() for z in stream.all_files()]))
- lockfile.close()
- return path
-
-@_deprecated('3.0', 'os.walk')
-def listdir_recursive(dirname, base=None, max_depth=None):
- """listdir_recursive(dirname, base, max_depth)
-
- Return a list of file and directory names found under dirname.
-
- If base is not None, prepend "{base}/" to each returned name.
-
- If max_depth is None, descend into directories and return only the
- names of files found in the directory tree.
-
- If max_depth is a non-negative integer, stop descending into
- directories at the given depth, and at that point return directory
- names instead.
-
- If max_depth==0 (and base is None) this is equivalent to
- sorted(os.listdir(dirname)).
- """
- allfiles = []
- for ent in sorted(os.listdir(dirname)):
- ent_path = os.path.join(dirname, ent)
- ent_base = os.path.join(base, ent) if base else ent
- if os.path.isdir(ent_path) and max_depth != 0:
- allfiles += listdir_recursive(
- ent_path, base=ent_base,
- max_depth=(max_depth-1 if max_depth else None))
- else:
- allfiles += [ent_base]
- return allfiles
class VersionChecker(object):
def assertVersionOutput(self, out, err):
- if sys.version_info >= (3, 0):
- self.assertEqual(err.getvalue(), '')
- v = out.getvalue()
- else:
- # Python 2 writes version info on stderr.
- self.assertEqual(out.getvalue(), '')
- v = err.getvalue()
+ self.assertEqual(err.getvalue(), '')
+ v = out.getvalue()
self.assertRegex(v, r"[0-9]+\.[0-9]+\.[0-9]+(\.dev[0-9]+)?$\n")
return self._resp_code
raise Exception
+
def mock_keep_responses(body, *codes, **headers):
"""Patch pycurl to return fake responses and raise exceptions.
return mock.patch('pycurl.Curl', cm)
-class MockStreamReader(object):
- def __init__(self, name='.', *data):
- self._name = name
- self._data = b''.join([
- b if isinstance(b, bytes) else b.encode()
- for b in data])
- self._data_locators = [str_keep_locator(d) for d in data]
- self.num_retries = 0
-
- def name(self):
- return self._name
-
- def readfrom(self, start, size, num_retries=None):
- return self._data[start:start + size]
-
class ApiClientMock(object):
def api_client_mock(self):
api_mock = mock.MagicMock(name='api_client_mock')
testfile.flush()
return testfile
-if sys.version_info < (3, 0):
- # There is no assert[Not]Regex that works in both Python 2 and 3,
- # so we backport Python 3 style to Python 2.
- def assertRegex(self, *args, **kwargs):
- return self.assertRegexpMatches(*args, **kwargs)
- def assertNotRegex(self, *args, **kwargs):
- return self.assertNotRegexpMatches(*args, **kwargs)
- unittest.TestCase.assertRegex = assertRegex
- unittest.TestCase.assertNotRegex = assertNotRegex
-
def binary_compare(a, b):
if len(a) != len(b):
return False
blocks_per_file=1,
files_per_stream=1,
streams=1):
- datablip = 'x' * bytes_per_block
+ datablip = b'x' * bytes_per_block
data_loc = tutil.str_keep_locator(datablip)
with tutil.mock_keep_responses(data_loc, 200):
- coll = arvados.CollectionWriter()
+ coll = arvados.collection.Collection()
for si in range(0, streams):
for fi in range(0, files_per_stream):
- with coll.open("stream{}/file{}.txt".format(si, fi)) as f:
+ with coll.open("stream{}/file{}.txt".format(si, fi), 'wb') as f:
for bi in range(0, blocks_per_file):
f.write(datablip)
return coll.manifest_text()
api_client,
normalize_api_kwargs,
api_kwargs_from_config,
- OrderedJsonModel,
_googleapiclient_log_lock,
)
from .arvados_testutil import fake_httplib2_response, mock_api_responses, queue_with
self.assertEqual(response.status, code)
self.assertEqual(response.get('status'), str(code))
- def test_ordered_json_model(self):
- mock_responses = {
- 'arvados.collections.get': (
- None,
- json.dumps(collections.OrderedDict(
- (c, int(c, 16)) for c in string.hexdigits
- )).encode(),
- ),
- }
- req_builder = apiclient_http.RequestMockBuilder(mock_responses)
- api = arvados.api('v1',
- requestBuilder=req_builder, model=OrderedJsonModel())
- result = api.collections().get(uuid='test').execute()
- self.assertEqual(string.hexdigits, ''.join(list(result.keys())))
-
def test_api_is_threadsafe(self):
api_kwargs = {
'host': os.environ['ARVADOS_API_HOST'],
with tutil.redirected_streams(stdout=out, stderr=out), \
self.assertRaises(SystemExit):
self.run_arv_keepdocker(['-x=unknown'], sys.stderr)
- self.assertRegex(out.getvalue(), 'unrecognized arguments')
+ self.assertRegex(out.getvalue(), r'unrecognized arguments')
def test_version_argument(self):
with tutil.redirected_streams(
self.assertEqual(out.getvalue(), '')
if expect_ok:
self.assertNotRegex(
- err.getvalue(), "refusing to store",
+ err.getvalue(), r"refusing to store",
msg=repr((supported, img_id)))
else:
self.assertRegex(
- err.getvalue(), "refusing to store",
+ err.getvalue(), r"refusing to store",
msg=repr((supported, img_id)))
if not supported:
self.assertRegex(
err.getvalue(),
- "server does not specify supported image formats",
+ r"server does not specify supported image formats",
msg=repr((supported, img_id)))
fakeDD = arvados.api('v1')._rootDesc
api()._rootDesc = fakeDD
self.run_arv_keepdocker(
['--force', '--force-image-format', 'testimage'], err)
- self.assertRegex(err.getvalue(), "forcing incompatible image")
+ self.assertRegex(err.getvalue(), r"forcing incompatible image")
def test_tag_given_twice(self):
with tutil.redirected_streams(stdout=tutil.StringIO, stderr=tutil.StringIO) as (out, err):
with self.assertRaises(SystemExit):
self.run_arv_keepdocker(['myrepo:mytag', 'extratag'], sys.stderr)
- self.assertRegex(err.getvalue(), "cannot add tag argument 'extratag'")
+ self.assertRegex(err.getvalue(), r"cannot add tag argument 'extratag'")
def test_image_given_as_repo_colon_tag(self):
with self.assertRaises(StopTest), \
out = tutil.StringIO()
with self.assertRaises(SystemExit):
self.run_arv_keepdocker([], sys.stderr, stdout=out)
- self.assertRegex(out.getvalue(), '\nregistry.example:1234/repo +latest ')
+ self.assertRegex(out.getvalue(), r'\nregistry.example:1234/repo +latest ')
finally:
api.links().delete(uuid=taglink['uuid']).execute()
self.assertEqual(1, error_mock.call_count)
def test_version_argument(self):
- if sys.version_info >= (3, 0):
- import warnings
- warnings.simplefilter("ignore")
+ import warnings
+ warnings.simplefilter("ignore")
with redirected_streams(stdout=StringIO, stderr=StringIO) as (out, err):
with self.assertRaises(SystemExit):
self.run_ls(['--version'], None)
from arvados.arvfile import ArvadosFile, ArvadosFileReader
from . import arvados_testutil as tutil
-from .test_stream import StreamFileReaderTestCase, StreamRetryTestMixin
+from .test_stream import StreamFileReaderTestMixin, StreamRetryTestMixin
class ArvadosFileWriterTestCase(unittest.TestCase):
class MockKeep(object):
self.assertEqual(b"01234567", keep.get("2e9ec317e197819358fbc43afca7d837+8"))
-class ArvadosFileReaderTestCase(StreamFileReaderTestCase):
+class ArvadosFileReaderTestCase(unittest.TestCase, StreamFileReaderTestMixin):
class MockParent(object):
class MockBlockMgr(object):
def __init__(self, blocks, nocache):
return ArvadosFileReaderTestCase.MockParent.MockBlockMgr(self.blocks, self.nocache)
+ def make_file_reader(self, name='emptyfile', data='', nocache=False):
+ loc = tutil.str_keep_locator(data)
+ af = ArvadosFile(ArvadosFileReaderTestCase.MockParent({loc: data}, nocache=nocache), name, stream=[Range(loc, 0, len(data))], segments=[Range(0, len(data), len(data))])
+ return ArvadosFileReader(af, mode='rb')
+
def make_count_reader(self, nocache=False):
stream = []
n = 0
blocks[loc] = d
stream.append(Range(loc, n, len(d)))
n += len(d)
- af = ArvadosFile(ArvadosFileReaderTestCase.MockParent(blocks, nocache), "count.txt", stream=stream, segments=[Range(1, 0, 3), Range(6, 3, 3), Range(11, 6, 3)])
+ af = ArvadosFile(ArvadosFileReaderTestCase.MockParent(blocks, nocache=nocache), "count.txt", stream=stream, segments=[Range(1, 0, 3), Range(6, 3, 3), Range(11, 6, 3)])
+ return ArvadosFileReader(af, mode="rb")
+
+ def make_newlines_reader(self, nocache=False):
+ stream = []
+ segments = []
+ n = 0
+ blocks = {}
+ for d in [b'one\ntwo\n\nth', b'ree\nfour\n\n']:
+ loc = tutil.str_keep_locator(d)
+ blocks[loc] = d
+ stream.append(Range(loc, n, len(d)))
+ segments.append(Range(n, len(d), n+len(d)))
+ n += len(d)
+ af = ArvadosFile(ArvadosFileReaderTestCase.MockParent(blocks, nocache=nocache), "count.txt", stream=stream, segments=segments)
return ArvadosFileReader(af, mode="rb")
def test_read_block_crossing_behavior(self):
sfile = self.make_count_reader(nocache=True)
self.assertEqual(b'12345678', sfile.read(8))
- def test_successive_reads(self):
- # Override StreamFileReaderTestCase.test_successive_reads
- sfile = self.make_count_reader(nocache=True)
- self.assertEqual(b'1234', sfile.read(4))
- self.assertEqual(b'5678', sfile.read(4))
- self.assertEqual(b'9', sfile.read(4))
- self.assertEqual(b'', sfile.read(4))
-
def test_tell_after_block_read(self):
- # Override StreamFileReaderTestCase.test_tell_after_block_read
sfile = self.make_count_reader(nocache=True)
self.assertEqual(b'12345678', sfile.read(8))
self.assertEqual(8, sfile.tell())
import parameterized
from unittest import mock
+from . import run_test_server
+from arvados._ranges import Range, LocatorAndRange, locators_and_ranges
+
import arvados
import arvados.keep
+
from arvados.collection import Collection, CollectionReader
from arvados._ranges import Range, LocatorAndRange
from . import arvados_testutil as tutil
from . import run_test_server
-class TestResumableWriter(arvados.ResumableCollectionWriter):
- KEEP_BLOCK_SIZE = 1024 # PUT to Keep every 1K.
-
- def current_state(self):
- return self.dump_state(copy.deepcopy)
-
-
@parameterized.parameterized_class([{"disk_cache": True}, {"disk_cache": False}])
class ArvadosCollectionsTest(run_test_server.TestCaseWithServers,
tutil.ArvadosBaseTestCase):
shutil.rmtree(cls._disk_cache_dir)
def write_foo_bar_baz(self):
- cw = arvados.CollectionWriter(self.api_client)
- self.assertEqual(cw.current_stream_name(), '.',
- 'current_stream_name() should be "." now')
- cw.set_current_file_name('foo.txt')
- cw.write(b'foo')
- self.assertEqual(cw.current_file_name(), 'foo.txt',
- 'current_file_name() should be foo.txt now')
- cw.start_new_file('bar.txt')
- cw.write(b'bar')
- cw.start_new_stream('baz')
- cw.write(b'baz')
- cw.set_current_file_name('baz.txt')
- self.assertEqual(cw.manifest_text(),
- ". 3858f62230ac3c915f300c664312c63f+6 0:3:foo.txt 3:3:bar.txt\n" +
- "./baz 73feffa4b7f6bb68e44cf984c85f6e88+3 0:3:baz.txt\n",
- "wrong manifest: got {}".format(cw.manifest_text()))
+ with arvados.collection.Collection(api_client=self.api_client).open('zzz', 'wb') as f:
+ f.write(b'foobar')
+ f.flush()
+ f.write(b'baz')
+ cw = arvados.collection.Collection(
+ api_client=self.api_client,
+ manifest_locator_or_text=
+ ". 3858f62230ac3c915f300c664312c63f+6 0:3:foo.txt 3:3:bar.txt\n" +
+ "./baz 73feffa4b7f6bb68e44cf984c85f6e88+3 0:3:baz.txt\n")
cw.save_new()
return cw.portable_data_hash()
'23ca013983d6239e98931cc779e68426+114',
'wrong locator hash: ' + self.write_foo_bar_baz())
- def test_local_collection_reader(self):
- foobarbaz = self.write_foo_bar_baz()
- cr = arvados.CollectionReader(
- foobarbaz + '+Xzizzle', self.api_client)
- got = []
- for s in cr.all_streams():
- for f in s.all_files():
- got += [[f.size(), f.stream_name(), f.name(), f.read(2**26)]]
- expected = [[3, '.', 'foo.txt', b'foo'],
- [3, '.', 'bar.txt', b'bar'],
- [3, './baz', 'baz.txt', b'baz']]
- self.assertEqual(got,
- expected)
- stream0 = cr.all_streams()[0]
- self.assertEqual(stream0.readfrom(0, 0),
- b'',
- 'reading zero bytes should have returned empty string')
- self.assertEqual(stream0.readfrom(0, 2**26),
- b'foobar',
- 'reading entire stream failed')
- self.assertEqual(stream0.readfrom(2**26, 0),
- b'',
- 'reading zero bytes should have returned empty string')
- self.assertEqual(3, len(cr))
- self.assertTrue(cr)
-
- def _test_subset(self, collection, expected):
- cr = arvados.CollectionReader(collection, self.api_client)
- for s in cr.all_streams():
- for ex in expected:
- if ex[0] == s:
- f = s.files()[ex[2]]
- got = [f.size(), f.stream_name(), f.name(), "".join(f.readall(2**26))]
- self.assertEqual(got,
- ex,
- 'all_files|as_manifest did not preserve manifest contents: got %s expected %s' % (got, ex))
-
- def test_collection_manifest_subset(self):
- foobarbaz = self.write_foo_bar_baz()
- self._test_subset(foobarbaz,
- [[3, '.', 'bar.txt', b'bar'],
- [3, '.', 'foo.txt', b'foo'],
- [3, './baz', 'baz.txt', b'baz']])
- self._test_subset((". %s %s 0:3:foo.txt 3:3:bar.txt\n" %
- (self.keep_client.put(b"foo"),
- self.keep_client.put(b"bar"))),
- [[3, '.', 'bar.txt', b'bar'],
- [3, '.', 'foo.txt', b'foo']])
- self._test_subset((". %s %s 0:2:fo.txt 2:4:obar.txt\n" %
- (self.keep_client.put(b"foo"),
- self.keep_client.put(b"bar"))),
- [[2, '.', 'fo.txt', b'fo'],
- [4, '.', 'obar.txt', b'obar']])
- self._test_subset((". %s %s 0:2:fo.txt 2:0:zero.txt 2:2:ob.txt 4:2:ar.txt\n" %
- (self.keep_client.put(b"foo"),
- self.keep_client.put(b"bar"))),
- [[2, '.', 'ar.txt', b'ar'],
- [2, '.', 'fo.txt', b'fo'],
- [2, '.', 'ob.txt', b'ob'],
- [0, '.', 'zero.txt', b'']])
-
def test_collection_empty_file(self):
- cw = arvados.CollectionWriter(self.api_client)
- cw.start_new_file('zero.txt')
- cw.write(b'')
+ cw = arvados.collection.Collection(api_client=self.api_client)
+ with cw.open('zero.txt', 'wb') as f:
+ pass
self.assertEqual(cw.manifest_text(), ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:zero.txt\n")
self.check_manifest_file_sizes(cw.manifest_text(), [0])
- cw = arvados.CollectionWriter(self.api_client)
- cw.start_new_file('zero.txt')
- cw.write(b'')
- cw.start_new_file('one.txt')
- cw.write(b'1')
- cw.start_new_stream('foo')
- cw.start_new_file('zero.txt')
- cw.write(b'')
- self.check_manifest_file_sizes(cw.manifest_text(), [0,1,0])
-
- def test_no_implicit_normalize(self):
- cw = arvados.CollectionWriter(self.api_client)
- cw.start_new_file('b')
- cw.write(b'b')
- cw.start_new_file('a')
- cw.write(b'')
- self.check_manifest_file_sizes(cw.manifest_text(), [1,0])
- self.check_manifest_file_sizes(
- arvados.CollectionReader(
- cw.manifest_text()).manifest_text(normalize=True),
- [0,1])
+
+ cw = arvados.collection.Collection(api_client=self.api_client)
+ with cw.open('zero.txt', 'wb') as f:
+ pass
+ with cw.open('one.txt', 'wb') as f:
+ f.write(b'1')
+ with cw.open('foo/zero.txt', 'wb') as f:
+ pass
+ # sorted, that's: [./one.txt, ./zero.txt, foo/zero.txt]
+ self.check_manifest_file_sizes(cw.manifest_text(), [1,0,0])
def check_manifest_file_sizes(self, manifest_text, expect_sizes):
- cr = arvados.CollectionReader(manifest_text, self.api_client)
got_sizes = []
- for f in cr.all_files():
- got_sizes += [f.size()]
+ def walk(subdir):
+ for fnm in subdir:
+ if isinstance(subdir[fnm], arvados.arvfile.ArvadosFile):
+ got_sizes.append(subdir[fnm].size())
+ else:
+ walk(subdir[fnm])
+ cr = arvados.CollectionReader(manifest_text, self.api_client)
+ walk(cr)
self.assertEqual(got_sizes, expect_sizes, "got wrong file sizes %s, expected %s" % (got_sizes, expect_sizes))
def test_normalized_collection(self):
Range('e', 40, 10),
Range('f', 50, 10)]
- self.assertEqual(arvados.locators_and_ranges(blocks2, 2, 2), [LocatorAndRange('a', 10, 2, 2)])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 12, 2), [LocatorAndRange('b', 10, 2, 2)])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 22, 2), [LocatorAndRange('c', 10, 2, 2)])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 32, 2), [LocatorAndRange('d', 10, 2, 2)])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 42, 2), [LocatorAndRange('e', 10, 2, 2)])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 52, 2), [LocatorAndRange('f', 10, 2, 2)])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 62, 2), [])
- self.assertEqual(arvados.locators_and_ranges(blocks2, -2, 2), [])
-
- self.assertEqual(arvados.locators_and_ranges(blocks2, 0, 2), [LocatorAndRange('a', 10, 0, 2)])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 10, 2), [LocatorAndRange('b', 10, 0, 2)])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 20, 2), [LocatorAndRange('c', 10, 0, 2)])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 30, 2), [LocatorAndRange('d', 10, 0, 2)])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 40, 2), [LocatorAndRange('e', 10, 0, 2)])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 50, 2), [LocatorAndRange('f', 10, 0, 2)])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 60, 2), [])
- self.assertEqual(arvados.locators_and_ranges(blocks2, -2, 2), [])
-
- self.assertEqual(arvados.locators_and_ranges(blocks2, 9, 2), [LocatorAndRange('a', 10, 9, 1), LocatorAndRange('b', 10, 0, 1)])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 19, 2), [LocatorAndRange('b', 10, 9, 1), LocatorAndRange('c', 10, 0, 1)])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 29, 2), [LocatorAndRange('c', 10, 9, 1), LocatorAndRange('d', 10, 0, 1)])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 39, 2), [LocatorAndRange('d', 10, 9, 1), LocatorAndRange('e', 10, 0, 1)])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 49, 2), [LocatorAndRange('e', 10, 9, 1), LocatorAndRange('f', 10, 0, 1)])
- self.assertEqual(arvados.locators_and_ranges(blocks2, 59, 2), [LocatorAndRange('f', 10, 9, 1)])
+ self.assertEqual(locators_and_ranges(blocks2, 2, 2), [LocatorAndRange('a', 10, 2, 2)])
+ self.assertEqual(locators_and_ranges(blocks2, 12, 2), [LocatorAndRange('b', 10, 2, 2)])
+ self.assertEqual(locators_and_ranges(blocks2, 22, 2), [LocatorAndRange('c', 10, 2, 2)])
+ self.assertEqual(locators_and_ranges(blocks2, 32, 2), [LocatorAndRange('d', 10, 2, 2)])
+ self.assertEqual(locators_and_ranges(blocks2, 42, 2), [LocatorAndRange('e', 10, 2, 2)])
+ self.assertEqual(locators_and_ranges(blocks2, 52, 2), [LocatorAndRange('f', 10, 2, 2)])
+ self.assertEqual(locators_and_ranges(blocks2, 62, 2), [])
+ self.assertEqual(locators_and_ranges(blocks2, -2, 2), [])
+
+ self.assertEqual(locators_and_ranges(blocks2, 0, 2), [LocatorAndRange('a', 10, 0, 2)])
+ self.assertEqual(locators_and_ranges(blocks2, 10, 2), [LocatorAndRange('b', 10, 0, 2)])
+ self.assertEqual(locators_and_ranges(blocks2, 20, 2), [LocatorAndRange('c', 10, 0, 2)])
+ self.assertEqual(locators_and_ranges(blocks2, 30, 2), [LocatorAndRange('d', 10, 0, 2)])
+ self.assertEqual(locators_and_ranges(blocks2, 40, 2), [LocatorAndRange('e', 10, 0, 2)])
+ self.assertEqual(locators_and_ranges(blocks2, 50, 2), [LocatorAndRange('f', 10, 0, 2)])
+ self.assertEqual(locators_and_ranges(blocks2, 60, 2), [])
+ self.assertEqual(locators_and_ranges(blocks2, -2, 2), [])
+
+ self.assertEqual(locators_and_ranges(blocks2, 9, 2), [LocatorAndRange('a', 10, 9, 1), LocatorAndRange('b', 10, 0, 1)])
+ self.assertEqual(locators_and_ranges(blocks2, 19, 2), [LocatorAndRange('b', 10, 9, 1), LocatorAndRange('c', 10, 0, 1)])
+ self.assertEqual(locators_and_ranges(blocks2, 29, 2), [LocatorAndRange('c', 10, 9, 1), LocatorAndRange('d', 10, 0, 1)])
+ self.assertEqual(locators_and_ranges(blocks2, 39, 2), [LocatorAndRange('d', 10, 9, 1), LocatorAndRange('e', 10, 0, 1)])
+ self.assertEqual(locators_and_ranges(blocks2, 49, 2), [LocatorAndRange('e', 10, 9, 1), LocatorAndRange('f', 10, 0, 1)])
+ self.assertEqual(locators_and_ranges(blocks2, 59, 2), [LocatorAndRange('f', 10, 9, 1)])
blocks3 = [Range('a', 0, 10),
Range('f', 50, 10),
Range('g', 60, 10)]
- self.assertEqual(arvados.locators_and_ranges(blocks3, 2, 2), [LocatorAndRange('a', 10, 2, 2)])
- self.assertEqual(arvados.locators_and_ranges(blocks3, 12, 2), [LocatorAndRange('b', 10, 2, 2)])
- self.assertEqual(arvados.locators_and_ranges(blocks3, 22, 2), [LocatorAndRange('c', 10, 2, 2)])
- self.assertEqual(arvados.locators_and_ranges(blocks3, 32, 2), [LocatorAndRange('d', 10, 2, 2)])
- self.assertEqual(arvados.locators_and_ranges(blocks3, 42, 2), [LocatorAndRange('e', 10, 2, 2)])
- self.assertEqual(arvados.locators_and_ranges(blocks3, 52, 2), [LocatorAndRange('f', 10, 2, 2)])
- self.assertEqual(arvados.locators_and_ranges(blocks3, 62, 2), [LocatorAndRange('g', 10, 2, 2)])
+ self.assertEqual(locators_and_ranges(blocks3, 2, 2), [LocatorAndRange('a', 10, 2, 2)])
+ self.assertEqual(locators_and_ranges(blocks3, 12, 2), [LocatorAndRange('b', 10, 2, 2)])
+ self.assertEqual(locators_and_ranges(blocks3, 22, 2), [LocatorAndRange('c', 10, 2, 2)])
+ self.assertEqual(locators_and_ranges(blocks3, 32, 2), [LocatorAndRange('d', 10, 2, 2)])
+ self.assertEqual(locators_and_ranges(blocks3, 42, 2), [LocatorAndRange('e', 10, 2, 2)])
+ self.assertEqual(locators_and_ranges(blocks3, 52, 2), [LocatorAndRange('f', 10, 2, 2)])
+ self.assertEqual(locators_and_ranges(blocks3, 62, 2), [LocatorAndRange('g', 10, 2, 2)])
blocks = [Range('a', 0, 10),
Range('b', 10, 15),
Range('c', 25, 5)]
- self.assertEqual(arvados.locators_and_ranges(blocks, 1, 0), [])
- self.assertEqual(arvados.locators_and_ranges(blocks, 0, 5), [LocatorAndRange('a', 10, 0, 5)])
- self.assertEqual(arvados.locators_and_ranges(blocks, 3, 5), [LocatorAndRange('a', 10, 3, 5)])
- self.assertEqual(arvados.locators_and_ranges(blocks, 0, 10), [LocatorAndRange('a', 10, 0, 10)])
-
- self.assertEqual(arvados.locators_and_ranges(blocks, 0, 11), [LocatorAndRange('a', 10, 0, 10),
- LocatorAndRange('b', 15, 0, 1)])
- self.assertEqual(arvados.locators_and_ranges(blocks, 1, 11), [LocatorAndRange('a', 10, 1, 9),
- LocatorAndRange('b', 15, 0, 2)])
- self.assertEqual(arvados.locators_and_ranges(blocks, 0, 25), [LocatorAndRange('a', 10, 0, 10),
- LocatorAndRange('b', 15, 0, 15)])
-
- self.assertEqual(arvados.locators_and_ranges(blocks, 0, 30), [LocatorAndRange('a', 10, 0, 10),
- LocatorAndRange('b', 15, 0, 15),
- LocatorAndRange('c', 5, 0, 5)])
- self.assertEqual(arvados.locators_and_ranges(blocks, 1, 30), [LocatorAndRange('a', 10, 1, 9),
- LocatorAndRange('b', 15, 0, 15),
- LocatorAndRange('c', 5, 0, 5)])
- self.assertEqual(arvados.locators_and_ranges(blocks, 0, 31), [LocatorAndRange('a', 10, 0, 10),
- LocatorAndRange('b', 15, 0, 15),
- LocatorAndRange('c', 5, 0, 5)])
-
- self.assertEqual(arvados.locators_and_ranges(blocks, 15, 5), [LocatorAndRange('b', 15, 5, 5)])
-
- self.assertEqual(arvados.locators_and_ranges(blocks, 8, 17), [LocatorAndRange('a', 10, 8, 2),
- LocatorAndRange('b', 15, 0, 15)])
-
- self.assertEqual(arvados.locators_and_ranges(blocks, 8, 20), [LocatorAndRange('a', 10, 8, 2),
- LocatorAndRange('b', 15, 0, 15),
- LocatorAndRange('c', 5, 0, 3)])
-
- self.assertEqual(arvados.locators_and_ranges(blocks, 26, 2), [LocatorAndRange('c', 5, 1, 2)])
-
- self.assertEqual(arvados.locators_and_ranges(blocks, 9, 15), [LocatorAndRange('a', 10, 9, 1),
- LocatorAndRange('b', 15, 0, 14)])
- self.assertEqual(arvados.locators_and_ranges(blocks, 10, 15), [LocatorAndRange('b', 15, 0, 15)])
- self.assertEqual(arvados.locators_and_ranges(blocks, 11, 15), [LocatorAndRange('b', 15, 1, 14),
- LocatorAndRange('c', 5, 0, 1)])
+ self.assertEqual(locators_and_ranges(blocks, 1, 0), [])
+ self.assertEqual(locators_and_ranges(blocks, 0, 5), [LocatorAndRange('a', 10, 0, 5)])
+ self.assertEqual(locators_and_ranges(blocks, 3, 5), [LocatorAndRange('a', 10, 3, 5)])
+ self.assertEqual(locators_and_ranges(blocks, 0, 10), [LocatorAndRange('a', 10, 0, 10)])
+
+ self.assertEqual(locators_and_ranges(blocks, 0, 11), [LocatorAndRange('a', 10, 0, 10),
+ LocatorAndRange('b', 15, 0, 1)])
+ self.assertEqual(locators_and_ranges(blocks, 1, 11), [LocatorAndRange('a', 10, 1, 9),
+ LocatorAndRange('b', 15, 0, 2)])
+ self.assertEqual(locators_and_ranges(blocks, 0, 25), [LocatorAndRange('a', 10, 0, 10),
+ LocatorAndRange('b', 15, 0, 15)])
+
+ self.assertEqual(locators_and_ranges(blocks, 0, 30), [LocatorAndRange('a', 10, 0, 10),
+ LocatorAndRange('b', 15, 0, 15),
+ LocatorAndRange('c', 5, 0, 5)])
+ self.assertEqual(locators_and_ranges(blocks, 1, 30), [LocatorAndRange('a', 10, 1, 9),
+ LocatorAndRange('b', 15, 0, 15),
+ LocatorAndRange('c', 5, 0, 5)])
+ self.assertEqual(locators_and_ranges(blocks, 0, 31), [LocatorAndRange('a', 10, 0, 10),
+ LocatorAndRange('b', 15, 0, 15),
+ LocatorAndRange('c', 5, 0, 5)])
+
+ self.assertEqual(locators_and_ranges(blocks, 15, 5), [LocatorAndRange('b', 15, 5, 5)])
+
+ self.assertEqual(locators_and_ranges(blocks, 8, 17), [LocatorAndRange('a', 10, 8, 2),
+ LocatorAndRange('b', 15, 0, 15)])
+
+ self.assertEqual(locators_and_ranges(blocks, 8, 20), [LocatorAndRange('a', 10, 8, 2),
+ LocatorAndRange('b', 15, 0, 15),
+ LocatorAndRange('c', 5, 0, 3)])
+
+ self.assertEqual(locators_and_ranges(blocks, 26, 2), [LocatorAndRange('c', 5, 1, 2)])
+
+ self.assertEqual(locators_and_ranges(blocks, 9, 15), [LocatorAndRange('a', 10, 9, 1),
+ LocatorAndRange('b', 15, 0, 14)])
+ self.assertEqual(locators_and_ranges(blocks, 10, 15), [LocatorAndRange('b', 15, 0, 15)])
+ self.assertEqual(locators_and_ranges(blocks, 11, 15), [LocatorAndRange('b', 15, 1, 14),
+ LocatorAndRange('c', 5, 0, 1)])
class MockKeep(object):
def __init__(self, content, num_retries=0):
def get(self, locator, num_retries=0, prefetch=False):
return self.content[locator]
- def test_stream_reader(self):
- keepblocks = {
- 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa+10': b'abcdefghij',
- 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb+15': b'klmnopqrstuvwxy',
- 'cccccccccccccccccccccccccccccccc+5': b'z0123',
- }
- mk = self.MockKeep(keepblocks)
-
- sr = arvados.StreamReader([".", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa+10", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb+15", "cccccccccccccccccccccccccccccccc+5", "0:30:foo"], mk)
-
- content = b'abcdefghijklmnopqrstuvwxyz0123456789'
-
- self.assertEqual(sr.readfrom(0, 30), content[0:30])
- self.assertEqual(sr.readfrom(2, 30), content[2:30])
-
- self.assertEqual(sr.readfrom(2, 8), content[2:10])
- self.assertEqual(sr.readfrom(0, 10), content[0:10])
-
- self.assertEqual(sr.readfrom(0, 5), content[0:5])
- self.assertEqual(sr.readfrom(5, 5), content[5:10])
- self.assertEqual(sr.readfrom(10, 5), content[10:15])
- self.assertEqual(sr.readfrom(15, 5), content[15:20])
- self.assertEqual(sr.readfrom(20, 5), content[20:25])
- self.assertEqual(sr.readfrom(25, 5), content[25:30])
- self.assertEqual(sr.readfrom(30, 5), b'')
-
def test_extract_file(self):
m1 = """. 5348b82a029fd9e971a811ce1f71360b+43 0:43:md5sum.txt
. 085c37f02916da1cad16f93c54d899b7+41 0:41:md6sum.txt
. 085c37f02916da1cad16f93c54d899b7+41 5348b82a029fd9e971a811ce1f71360b+43 8b22da26f9f433dea0a10e5ec66d73ba+43 47:80:md8sum.txt
. 085c37f02916da1cad16f93c54d899b7+41 5348b82a029fd9e971a811ce1f71360b+43 8b22da26f9f433dea0a10e5ec66d73ba+43 40:80:md9sum.txt
"""
-
- m2 = arvados.CollectionReader(m1, self.api_client).manifest_text(normalize=True)
-
+ coll = arvados.CollectionReader(m1, self.api_client)
+ m2 = coll.manifest_text(normalize=True)
self.assertEqual(m2,
". 5348b82a029fd9e971a811ce1f71360b+43 085c37f02916da1cad16f93c54d899b7+41 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md5sum.txt 43:41:md6sum.txt 84:43:md7sum.txt 6:37:md8sum.txt 84:43:md8sum.txt 83:1:md9sum.txt 0:43:md9sum.txt 84:36:md9sum.txt\n")
- files = arvados.CollectionReader(
- m2, self.api_client).all_streams()[0].files()
-
- self.assertEqual(files['md5sum.txt'].as_manifest(),
+ self.assertEqual(coll['md5sum.txt'].manifest_text(),
". 5348b82a029fd9e971a811ce1f71360b+43 0:43:md5sum.txt\n")
- self.assertEqual(files['md6sum.txt'].as_manifest(),
+ self.assertEqual(coll['md6sum.txt'].manifest_text(),
". 085c37f02916da1cad16f93c54d899b7+41 0:41:md6sum.txt\n")
- self.assertEqual(files['md7sum.txt'].as_manifest(),
+ self.assertEqual(coll['md7sum.txt'].manifest_text(),
". 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md7sum.txt\n")
- self.assertEqual(files['md9sum.txt'].as_manifest(),
+ self.assertEqual(coll['md9sum.txt'].manifest_text(),
". 085c37f02916da1cad16f93c54d899b7+41 5348b82a029fd9e971a811ce1f71360b+43 8b22da26f9f433dea0a10e5ec66d73ba+43 40:80:md9sum.txt\n")
- def test_write_directory_tree(self):
- cwriter = arvados.CollectionWriter(self.api_client)
- cwriter.write_directory_tree(self.build_directory_tree(
- ['basefile', 'subdir/subfile']))
- self.assertEqual(cwriter.manifest_text(),
- """. c5110c5ac93202d8e0f9e381f22bac0f+8 0:8:basefile
-./subdir 1ca4dec89403084bf282ad31e6cf7972+14 0:14:subfile\n""")
-
- def test_write_named_directory_tree(self):
- cwriter = arvados.CollectionWriter(self.api_client)
- cwriter.write_directory_tree(self.build_directory_tree(
- ['basefile', 'subdir/subfile']), 'root')
- self.assertEqual(
- cwriter.manifest_text(),
- """./root c5110c5ac93202d8e0f9e381f22bac0f+8 0:8:basefile
-./root/subdir 1ca4dec89403084bf282ad31e6cf7972+14 0:14:subfile\n""")
-
- def test_write_directory_tree_in_one_stream(self):
- cwriter = arvados.CollectionWriter(self.api_client)
- cwriter.write_directory_tree(self.build_directory_tree(
- ['basefile', 'subdir/subfile']), max_manifest_depth=0)
- self.assertEqual(cwriter.manifest_text(),
- """. 4ace875ffdc6824a04950f06858f4465+22 0:8:basefile 8:14:subdir/subfile\n""")
-
- def test_write_directory_tree_with_limited_recursion(self):
- cwriter = arvados.CollectionWriter(self.api_client)
- cwriter.write_directory_tree(
- self.build_directory_tree(['f1', 'd1/f2', 'd1/d2/f3']),
- max_manifest_depth=1)
- self.assertEqual(cwriter.manifest_text(),
- """. bd19836ddb62c11c55ab251ccaca5645+2 0:2:f1
-./d1 50170217e5b04312024aa5cd42934494+13 0:8:d2/f3 8:5:f2\n""")
-
- def test_write_directory_tree_with_zero_recursion(self):
- cwriter = arvados.CollectionWriter(self.api_client)
- content = 'd1/d2/f3d1/f2f1'
- blockhash = tutil.str_keep_locator(content)
- cwriter.write_directory_tree(
- self.build_directory_tree(['f1', 'd1/f2', 'd1/d2/f3']),
- max_manifest_depth=0)
- self.assertEqual(
- cwriter.manifest_text(),
- ". {} 0:8:d1/d2/f3 8:5:d1/f2 13:2:f1\n".format(blockhash))
-
- def test_write_one_file(self):
- cwriter = arvados.CollectionWriter(self.api_client)
- with self.make_test_file() as testfile:
- cwriter.write_file(testfile.name)
- self.assertEqual(
- cwriter.manifest_text(),
- ". 098f6bcd4621d373cade4e832627b4f6+4 0:4:{}\n".format(
- os.path.basename(testfile.name)))
-
- def test_write_named_file(self):
- cwriter = arvados.CollectionWriter(self.api_client)
- with self.make_test_file() as testfile:
- cwriter.write_file(testfile.name, 'foo')
- self.assertEqual(cwriter.manifest_text(),
- ". 098f6bcd4621d373cade4e832627b4f6+4 0:4:foo\n")
-
- def test_write_multiple_files(self):
- cwriter = arvados.CollectionWriter(self.api_client)
- for letter in 'ABC':
- with self.make_test_file(letter.encode()) as testfile:
- cwriter.write_file(testfile.name, letter)
- self.assertEqual(
- cwriter.manifest_text(),
- ". 902fbdd2b1df0c4f70b4a5d23525e932+3 0:1:A 1:1:B 2:1:C\n")
-
- def test_basic_resume(self):
- cwriter = TestResumableWriter()
- with self.make_test_file() as testfile:
- cwriter.write_file(testfile.name, 'test')
- resumed = TestResumableWriter.from_state(cwriter.current_state())
- self.assertEqual(cwriter.manifest_text(), resumed.manifest_text(),
- "resumed CollectionWriter had different manifest")
-
- def test_resume_fails_when_missing_dependency(self):
- cwriter = TestResumableWriter()
- with self.make_test_file() as testfile:
- cwriter.write_file(testfile.name, 'test')
- self.assertRaises(arvados.errors.StaleWriterStateError,
- TestResumableWriter.from_state,
- cwriter.current_state())
-
- def test_resume_fails_when_dependency_mtime_changed(self):
- cwriter = TestResumableWriter()
- with self.make_test_file() as testfile:
- cwriter.write_file(testfile.name, 'test')
- os.utime(testfile.name, (0, 0))
- self.assertRaises(arvados.errors.StaleWriterStateError,
- TestResumableWriter.from_state,
- cwriter.current_state())
-
- def test_resume_fails_when_dependency_is_nonfile(self):
- cwriter = TestResumableWriter()
- cwriter.write_file('/dev/null', 'empty')
- self.assertRaises(arvados.errors.StaleWriterStateError,
- TestResumableWriter.from_state,
- cwriter.current_state())
-
- def test_resume_fails_when_dependency_size_changed(self):
- cwriter = TestResumableWriter()
- with self.make_test_file() as testfile:
- cwriter.write_file(testfile.name, 'test')
- orig_mtime = os.fstat(testfile.fileno()).st_mtime
- testfile.write(b'extra')
- testfile.flush()
- os.utime(testfile.name, (orig_mtime, orig_mtime))
- self.assertRaises(arvados.errors.StaleWriterStateError,
- TestResumableWriter.from_state,
- cwriter.current_state())
-
- def test_resume_fails_with_expired_locator(self):
- cwriter = TestResumableWriter()
- state = cwriter.current_state()
- # Add an expired locator to the state.
- state['_current_stream_locators'].append(''.join([
- 'a' * 32, '+1+A', 'b' * 40, '@', '10000000']))
- self.assertRaises(arvados.errors.StaleWriterStateError,
- TestResumableWriter.from_state, state)
-
- def test_arbitrary_objects_not_resumable(self):
- cwriter = TestResumableWriter()
- with open('/dev/null') as badfile:
- self.assertRaises(arvados.errors.AssertionError,
- cwriter.write_file, badfile)
-
- def test_arbitrary_writes_not_resumable(self):
- cwriter = TestResumableWriter()
- self.assertRaises(arvados.errors.AssertionError,
- cwriter.write, "badtext")
-
class CollectionTestMixin(tutil.ApiClientMock):
API_COLLECTIONS = run_test_server.fixture('collections')
reader = arvados.CollectionReader(self.DEFAULT_UUID, api_client=client,
num_retries=3)
with tutil.mock_keep_responses('foo', 500, 500, 200):
- self.assertEqual(b'foo',
- b''.join(f.read(9) for f in reader.all_files()))
+ self.assertEqual('foo', reader.open('foo', 'r').read())
def test_read_nonnormalized_manifest_with_collection_reader(self):
# client should be able to use CollectionReader on a manifest without normalizing it
reader.stripped_manifest())
# Ensure stripped_manifest() didn't mutate our reader.
self.assertEqual(nonnormal, reader.manifest_text())
- # Ensure the files appear in the order given in the manifest.
- self.assertEqual(
- [[6, '.', 'foo.txt'],
- [0, '.', 'bar.txt']],
- [[f.size(), f.stream_name(), f.name()]
- for f in reader.all_streams()[0].all_files()])
def test_read_empty_collection(self):
client = self.api_client_mock(200)
self.assertRaises(IOError, reader.open, 'nonexistent')
-@unittest.skip("will be removed in #15397")
-class CollectionWriterTestCase(unittest.TestCase, CollectionTestMixin):
- def mock_keep(self, body, *codes, **headers):
- headers.setdefault('x-keep-replicas-stored', 2)
- return tutil.mock_keep_responses(body, *codes, **headers)
-
- def foo_writer(self, **kwargs):
- kwargs.setdefault('api_client', self.api_client_mock())
- writer = arvados.CollectionWriter(**kwargs)
- writer.start_new_file('foo')
- writer.write(b'foo')
- return writer
-
- def test_write_whole_collection(self):
- writer = self.foo_writer()
- with self.mock_keep(self.DEFAULT_DATA_HASH, 200, 200):
- self.assertEqual(self.DEFAULT_DATA_HASH, writer.finish())
-
- def test_write_no_default(self):
- writer = self.foo_writer()
- with self.mock_keep(None, 500):
- with self.assertRaises(arvados.errors.KeepWriteError):
- writer.finish()
-
- def test_write_insufficient_replicas_via_proxy(self):
- writer = self.foo_writer(replication=3)
- with self.mock_keep(None, 200, **{'x-keep-replicas-stored': 2}):
- with self.assertRaises(arvados.errors.KeepWriteError):
- writer.manifest_text()
-
- def test_write_insufficient_replicas_via_disks(self):
- client = mock.MagicMock(name='api_client')
- with self.mock_keep(
- None, 200, 200,
- **{'x-keep-replicas-stored': 1}) as keepmock:
- self.mock_keep_services(client, status=200, service_type='disk', count=2)
- writer = self.foo_writer(api_client=client, replication=3)
- with self.assertRaises(arvados.errors.KeepWriteError):
- writer.manifest_text()
-
- def test_write_three_replicas(self):
- client = mock.MagicMock(name='api_client')
- with self.mock_keep(
- "", 500, 500, 500, 200, 200, 200,
- **{'x-keep-replicas-stored': 1}) as keepmock:
- self.mock_keep_services(client, status=200, service_type='disk', count=6)
- writer = self.foo_writer(api_client=client, replication=3)
- writer.manifest_text()
- self.assertEqual(6, keepmock.call_count)
-
- def test_write_whole_collection_through_retries(self):
- writer = self.foo_writer(num_retries=2)
- with self.mock_keep(self.DEFAULT_DATA_HASH,
- 500, 500, 200, 500, 500, 200):
- self.assertEqual(self.DEFAULT_DATA_HASH, writer.finish())
-
- def test_flush_data_retries(self):
- writer = self.foo_writer(num_retries=2)
- foo_hash = self.DEFAULT_MANIFEST.split()[1]
- with self.mock_keep(foo_hash, 500, 200):
- writer.flush_data()
- self.assertEqual(self.DEFAULT_MANIFEST, writer.manifest_text())
-
- def test_one_open(self):
- client = self.api_client_mock()
- writer = arvados.CollectionWriter(client)
- with writer.open('out') as out_file:
- self.assertEqual('.', writer.current_stream_name())
- self.assertEqual('out', writer.current_file_name())
- out_file.write(b'test data')
- data_loc = tutil.str_keep_locator('test data')
- self.assertTrue(out_file.closed, "writer file not closed after context")
- self.assertRaises(ValueError, out_file.write, 'extra text')
- with self.mock_keep(data_loc, 200) as keep_mock:
- self.assertEqual(". {} 0:9:out\n".format(data_loc),
- writer.manifest_text())
-
- def test_open_writelines(self):
- client = self.api_client_mock()
- writer = arvados.CollectionWriter(client)
- with writer.open('six') as out_file:
- out_file.writelines(['12', '34', '56'])
- data_loc = tutil.str_keep_locator('123456')
- with self.mock_keep(data_loc, 200) as keep_mock:
- self.assertEqual(". {} 0:6:six\n".format(data_loc),
- writer.manifest_text())
-
- def test_open_flush(self):
- client = self.api_client_mock()
- data_loc1 = tutil.str_keep_locator('flush1')
- data_loc2 = tutil.str_keep_locator('flush2')
- with self.mock_keep((data_loc1, 200), (data_loc2, 200)) as keep_mock:
- writer = arvados.CollectionWriter(client)
- with writer.open('flush_test') as out_file:
- out_file.write(b'flush1')
- out_file.flush()
- out_file.write(b'flush2')
- self.assertEqual(". {} {} 0:12:flush_test\n".format(data_loc1,
- data_loc2),
- writer.manifest_text())
-
- def test_two_opens_same_stream(self):
- client = self.api_client_mock()
- writer = arvados.CollectionWriter(client)
- with writer.open('.', '1') as out_file:
- out_file.write(b'1st')
- with writer.open('.', '2') as out_file:
- out_file.write(b'2nd')
- data_loc = tutil.str_keep_locator('1st2nd')
- with self.mock_keep(data_loc, 200) as keep_mock:
- self.assertEqual(". {} 0:3:1 3:3:2\n".format(data_loc),
- writer.manifest_text())
-
- def test_two_opens_two_streams(self):
- client = self.api_client_mock()
- data_loc1 = tutil.str_keep_locator('file')
- data_loc2 = tutil.str_keep_locator('indir')
- with self.mock_keep((data_loc1, 200), (data_loc2, 200)) as keep_mock:
- writer = arvados.CollectionWriter(client)
- with writer.open('file') as out_file:
- out_file.write(b'file')
- with writer.open('./dir', 'indir') as out_file:
- out_file.write(b'indir')
- expected = ". {} 0:4:file\n./dir {} 0:5:indir\n".format(
- data_loc1, data_loc2)
- self.assertEqual(expected, writer.manifest_text())
-
- def test_dup_open_fails(self):
- client = self.api_client_mock()
- writer = arvados.CollectionWriter(client)
- file1 = writer.open('one')
- self.assertRaises(arvados.errors.AssertionError, writer.open, 'two')
-
-
class CollectionMethods(run_test_server.TestCaseWithServers):
def test_keys_values_items_support_indexing(self):
with c.open('bar', 'wb') as f:
f.write(b'bar')
self.assertEqual(2, len(c.keys()))
- if sys.version_info < (3, 0):
- # keys() supports indexing only for python2 callers.
- fn0 = c.keys()[0]
- fn1 = c.keys()[1]
- else:
- fn0, fn1 = c.keys()
+ fn0, fn1 = c.keys()
self.assertEqual(2, len(c.values()))
f0 = c.values()[0]
f1 = c.values()[1]
def setUp(self):
arvados.config.KEEP_BLOCK_SIZE = 4
- if sys.version_info < (3, 0):
- import unicodedata
- self.sailboat = unicodedata.lookup('SAILBOAT')
- self.snowman = unicodedata.lookup('SNOWMAN')
- else:
- self.sailboat = '\N{SAILBOAT}'
- self.snowman = '\N{SNOWMAN}'
+ self.sailboat = '\N{SAILBOAT}'
+ self.snowman = '\N{SNOWMAN}'
def tearDown(self):
arvados.config.KEEP_BLOCK_SIZE = 2 ** 26
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-
-import arvados.crunch
-import os
-import shutil
-import tempfile
-import unittest
-
-class TaskOutputDirTest(unittest.TestCase):
- def setUp(self):
- self.tmp = tempfile.mkdtemp()
- os.environ['TASK_KEEPMOUNT_TMP'] = self.tmp
-
- def tearDown(self):
- os.environ.pop('TASK_KEEPMOUNT_TMP')
- shutil.rmtree(self.tmp)
-
- def test_env_var(self):
- out = arvados.crunch.TaskOutputDir()
- self.assertEqual(out.path, self.tmp)
-
- with open(os.path.join(self.tmp, '.arvados#collection'), 'w') as f:
- f.write('{\n "manifest_text":"",\n "uuid":null\n}\n')
- self.assertEqual(out.manifest_text(), '')
-
- # Special file must be re-read on each call to manifest_text().
- with open(os.path.join(self.tmp, '.arvados#collection'), 'w') as f:
- f.write(r'{"manifest_text":". unparsed 0:3:foo\n","uuid":null}')
- self.assertEqual(out.manifest_text(), ". unparsed 0:3:foo\n")
foo_locator = self.keep_client.put('foo')
self.assertRegex(
foo_locator,
- '^acbd18db4cc2f85cedef654fccc4a4d8\+3',
+ r'^acbd18db4cc2f85cedef654fccc4a4d8\+3',
'wrong md5 hash from Keep.put("foo"): ' + foo_locator)
# 6 bytes because uploaded 2 copies
blob_locator = self.keep_client.put(blob_str)
self.assertRegex(
blob_locator,
- '^7fc7c53b45e53926ba52821140fef396\+6',
+ r'^7fc7c53b45e53926ba52821140fef396\+6',
('wrong locator from Keep.put(<binarydata>):' + blob_locator))
self.assertEqual(self.keep_client.get(blob_locator),
blob_str,
blob_locator = self.keep_client.put(blob_data)
self.assertRegex(
blob_locator,
- '^84d90fc0d8175dd5dcfab04b999bc956\+67108864',
+ r'^84d90fc0d8175dd5dcfab04b999bc956\+67108864',
('wrong locator from Keep.put(<binarydata>): ' + blob_locator))
self.assertEqual(self.keep_client.get(blob_locator),
blob_data,
blob_locator = self.keep_client.put(blob_data, copies=1)
self.assertRegex(
blob_locator,
- '^c902006bc98a3eb4a3663b65ab4a6fab\+8',
+ r'^c902006bc98a3eb4a3663b65ab4a6fab\+8',
('wrong locator from Keep.put(<binarydata>): ' + blob_locator))
self.assertEqual(self.keep_client.get(blob_locator),
blob_data,
blob_locator = self.keep_client.put('', copies=1)
self.assertRegex(
blob_locator,
- '^d41d8cd98f00b204e9800998ecf8427e\+0',
+ r'^d41d8cd98f00b204e9800998ecf8427e\+0',
('wrong locator from Keep.put(""): ' + blob_locator))
- def test_unicode_must_be_ascii(self):
- # If unicode type, must only consist of valid ASCII
- foo_locator = self.keep_client.put(u'foo')
- self.assertRegex(
- foo_locator,
- '^acbd18db4cc2f85cedef654fccc4a4d8\+3',
- 'wrong md5 hash from Keep.put("foo"): ' + foo_locator)
-
- if sys.version_info < (3, 0):
- with self.assertRaises(UnicodeEncodeError):
- # Error if it is not ASCII
- self.keep_client.put(u'\xe2')
-
+ def test_KeepPutDataType(self):
with self.assertRaises(AttributeError):
# Must be bytes or have an encode() method
self.keep_client.put({})
locator = self.keep_client.put('test_head')
self.assertRegex(
locator,
- '^b9a772c7049325feb7130fff1f8333e9\+9',
+ r'^b9a772c7049325feb7130fff1f8333e9\+9',
'wrong md5 hash from Keep.put for "test_head": ' + locator)
self.assertEqual(True, self.keep_client.head(locator))
self.assertEqual(self.keep_client.get(locator),
# GET from a different user => bad request
run_test_server.authorize_with('spectator')
+ keep_client2 = arvados.KeepClient(block_cache=self.make_block_cache(self.disk_cache))
self.assertRaises(arvados.errors.KeepReadError,
- arvados.Keep.get,
+ keep_client2.get,
bar_locator)
# Unauthenticated GET for a signed locator => bad request
baz_locator = keep_client.put('baz')
self.assertRegex(
baz_locator,
- '^73feffa4b7f6bb68e44cf984c85f6e88\+3',
+ r'^73feffa4b7f6bb68e44cf984c85f6e88\+3',
'wrong md5 hash from Keep.put("baz"): ' + baz_locator)
self.assertEqual(keep_client.get(baz_locator),
b'baz',
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: Apache-2.0
-
-import os
-import unittest
-
-from unittest import mock
-
-import arvados
-import arvados.collection
-
-class TestSDK(unittest.TestCase):
-
- @mock.patch('arvados.current_task')
- @mock.patch('arvados.current_job')
- def test_one_task_per_input_file_normalize(self, mock_job, mock_task):
- mock_api = mock.MagicMock()
-
- # This manifest will be reduced from three lines to one when it is
- # normalized.
- nonnormalized_manifest = """. 5348b82a029fd9e971a811ce1f71360b+43 0:43:md5sum.txt
-. 085c37f02916da1cad16f93c54d899b7+41 0:41:md5sum.txt
-. 8b22da26f9f433dea0a10e5ec66d73ba+43 0:43:md5sum.txt
-"""
- dummy_hash = 'ffffffffffffffffffffffffffffffff+0'
-
- mock_job.return_value = {
- 'uuid': 'none',
- 'script_parameters': {
- 'input': dummy_hash
- }
- }
- mock_task.return_value = {
- 'uuid': 'none',
- 'sequence': 0,
- }
- # mock the API client to return a collection with a nonnormalized manifest.
- mock_api.collections().get().execute.return_value = {
- 'uuid': 'zzzzz-4zz18-mockcollection0',
- 'portable_data_hash': dummy_hash,
- 'manifest_text': nonnormalized_manifest,
- }
-
- # Because one_task_per_input_file normalizes this collection,
- # it should now create only one job task and not three.
- arvados.job_setup.one_task_per_input_file(and_end_task=False, api_client=mock_api)
- mock_api.job_tasks().create().execute.assert_called_once_with()
from unittest import mock
import arvados
-from arvados import StreamReader, StreamFileReader
from arvados._ranges import Range
from . import arvados_testutil as tutil
from . import run_test_server
-class StreamFileReaderTestCase(unittest.TestCase):
- def make_count_reader(self):
- stream = tutil.MockStreamReader('.', '01234', '34567', '67890')
- return StreamFileReader(stream, [Range(1, 0, 3), Range(6, 3, 3), Range(11, 6, 3)],
- 'count.txt')
-
+class StreamFileReaderTestMixin(object):
def test_read_block_crossing_behavior(self):
# read() calls will be aligned on block boundaries - see #3663.
sfile = self.make_count_reader()
def test_successive_reads(self):
sfile = self.make_count_reader()
- for expect in [b'123', b'456', b'789', b'']:
- self.assertEqual(expect, sfile.read(10))
+ for expect in [b'1234', b'5678', b'9', b'']:
+ self.assertEqual(expect, sfile.read(4))
def test_readfrom_spans_blocks(self):
sfile = self.make_count_reader()
def test_size(self):
self.assertEqual(9, self.make_count_reader().size())
- def test_tell_after_block_read(self):
- sfile = self.make_count_reader()
- sfile.read(5)
- self.assertEqual(3, sfile.tell())
-
def test_tell_after_small_read(self):
sfile = self.make_count_reader()
sfile.read(1)
self.assertEqual(b'12', sfile.read(2))
self.assertTrue(sfile.closed, "reader is open after context")
- def make_newlines_reader(self):
- stream = tutil.MockStreamReader('.', 'one\ntwo\n\nth', 'ree\nfour\n\n')
- return StreamFileReader(stream, [Range(0, 0, 11), Range(11, 11, 10)], 'count.txt')
-
def check_lines(self, actual):
self.assertEqual(['one\n', 'two\n', '\n', 'three\n', 'four\n', '\n'],
actual)
def test_readlines_sizehint(self):
result = self.make_newlines_reader().readlines(8)
- self.assertEqual(['one\n', 'two\n'], result[:2])
- self.assertNotIn('three\n', result)
+ self.assertEqual(['one\n', 'two\n', '\n', 'three\n', 'four\n', '\n'], result)
def test_name_attribute(self):
- # Test both .name and .name() (for backward compatibility)
- stream = tutil.MockStreamReader()
- sfile = StreamFileReader(stream, [Range(0, 0, 0)], 'nametest')
+ sfile = self.make_file_reader(name='nametest')
self.assertEqual('nametest', sfile.name)
- self.assertEqual('nametest', sfile.name())
def check_decompressed_name(self, filename, expect):
- stream = tutil.MockStreamReader('.', '')
- reader = StreamFileReader(stream, [Range(0, 0, 0)], filename)
+ reader = self.make_file_reader(name=filename)
self.assertEqual(expect, reader.decompressed_name())
def test_decompressed_name_uncompressed_file(self):
def check_decompression(self, compress_ext, compress_func):
test_text = b'decompression\ntest\n'
test_data = compress_func(test_text)
- stream = tutil.MockStreamReader('.', test_data)
- reader = StreamFileReader(stream, [Range(0, 0, len(test_data))],
- 'test.' + compress_ext)
+ reader = self.make_file_reader(name='test.'+compress_ext, data=test_data)
self.assertEqual(test_text, b''.join(reader.readall_decompressed()))
@staticmethod
self.read_for_test(reader, 10, num_retries=1)
-class StreamReaderTestCase(unittest.TestCase, StreamRetryTestMixin):
- def reader_for(self, coll_name, **kwargs):
- return StreamReader(self.manifest_for(coll_name).split(),
- self.keep_client(), **kwargs)
-
- def read_for_test(self, reader, byte_count, **kwargs):
- return reader.readfrom(0, byte_count, **kwargs)
-
- def test_manifest_text_without_keep_client(self):
- mtext = self.manifest_for('multilevel_collection_1')
- for line in mtext.rstrip('\n').split('\n'):
- reader = StreamReader(line.split())
- self.assertEqual(line + '\n', reader.manifest_text())
-
-
-class StreamFileReadTestCase(unittest.TestCase, StreamRetryTestMixin):
- def reader_for(self, coll_name, **kwargs):
- return StreamReader(self.manifest_for(coll_name).split(),
- self.keep_client(), **kwargs).all_files()[0]
-
- def read_for_test(self, reader, byte_count, **kwargs):
- return reader.read(byte_count, **kwargs)
-
-
-class StreamFileReadFromTestCase(StreamFileReadTestCase):
- def read_for_test(self, reader, byte_count, **kwargs):
- return reader.readfrom(0, byte_count, **kwargs)
-
-
-class StreamFileReadAllTestCase(StreamFileReadTestCase):
- def read_for_test(self, reader, byte_count, **kwargs):
- return b''.join(reader.readall(**kwargs))
-
-
-class StreamFileReadAllDecompressedTestCase(StreamFileReadTestCase):
- def read_for_test(self, reader, byte_count, **kwargs):
- return b''.join(reader.readall_decompressed(**kwargs))
-
-
-class StreamFileReadlinesTestCase(StreamFileReadTestCase):
- def read_for_test(self, reader, byte_count, **kwargs):
- return ''.join(reader.readlines(**kwargs)).encode()
-
if __name__ == '__main__':
unittest.main()
import arvados
import arvados.util
-class MkdirDashPTest(unittest.TestCase):
- def setUp(self):
- try:
- os.path.mkdir('./tmp')
- except:
- pass
- def tearDown(self):
- try:
- os.unlink('./tmp/bar')
- os.rmdir('./tmp/foo')
- os.rmdir('./tmp')
- except:
- pass
- def runTest(self):
- arvados.util.mkdir_dash_p('./tmp/foo')
- with open('./tmp/bar', 'wb') as f:
- f.write(b'bar')
- self.assertRaises(OSError, arvados.util.mkdir_dash_p, './tmp/bar')
-
-
-class RunCommandTestCase(unittest.TestCase):
- def test_success(self):
- stdout, stderr = arvados.util.run_command(['echo', 'test'],
- stderr=subprocess.PIPE)
- self.assertEqual("test\n".encode(), stdout)
- self.assertEqual("".encode(), stderr)
-
- def test_failure(self):
- with self.assertRaises(arvados.errors.CommandFailedError):
- arvados.util.run_command(['false'])
class KeysetTestHelper:
def __init__(self, expect):
require 'arvados'
arv = Arvados.new( { :suppress_ssl_warnings => false } )
-pt_list = arv.pipeline_template.list(where:{})
-puts pt_list[:items].first.inspect
+cr_list = arv.container_request.list(where:{})
+puts cr_list[:items].first.inspect
-pt = arv.pipeline_template.get(uuid:"9zb4a-p5p6p-fkkbrl98u3pk87m")
+cr = arv.container_request.get(uuid:"zzzzz-xvhdp-fkkbrl98u3pk87m")
puts pt.inspect
# translate UUID to numeric ID here.
resource_attrs[:user_id] =
User.where(uuid: resource_attrs.delete(:owner_uuid)).first.andand.id
- elsif not resource_attrs[:user_id]
+ else
resource_attrs[:user_id] = current_user.id
end
resource_attrs[:api_client_id] = Thread.current[:api_client].id
protected
+ ActionNameMap = {
+ 'destroy' => 'delete',
+ 'index' => 'list',
+ 'show' => 'get',
+ }
+
def discovery_doc
Rails.application.eager_load!
remoteHosts = {}
"https://api.arvados.org/auth/arvados.readonly"
]
},
- index: {
- id: "arvados.#{k.to_s.underscore.pluralize}.index",
+ list: {
+ id: "arvados.#{k.to_s.underscore.pluralize}.list",
path: k.to_s.underscore.pluralize,
httpMethod: "GET",
description:
- %|Index #{k.to_s.pluralize}.
+ %|List #{k.to_s.pluralize}.
- The <code>index</code> method returns a
+ The <code>list</code> method returns a
<a href="/api/resources.html">resource list</a> of
matching #{k.to_s.pluralize}. For example:
httpMethod = ['GET', 'POST', 'PUT', 'DELETE'].map { |method|
method if route.verb.match(method)
}.compact.first
- if httpMethod and
- route.defaults[:controller] == 'arvados/v1/' + k.to_s.underscore.pluralize and
- ctl_class.action_methods.include? action
- if !d_methods[action.to_sym]
+ if httpMethod &&
+ route.defaults[:controller] == 'arvados/v1/' + k.to_s.underscore.pluralize &&
+ ctl_class.action_methods.include?(action)
+ method_name = ActionNameMap[action] || action
+ method_key = method_name.to_sym
+ if !d_methods[method_key]
method = {
- id: "arvados.#{k.to_s.underscore.pluralize}.#{action}",
+ id: "arvados.#{k.to_s.underscore.pluralize}.#{method_name}",
path: route.path.spec.to_s.sub('/arvados/v1/','').sub('(.:format)','').sub(/:(uu)?id/,'{uuid}'),
httpMethod: httpMethod,
- description: "#{action} #{k.to_s.underscore.pluralize}",
+ description: "#{method_name} #{k.to_s.underscore.pluralize}",
parameters: {},
response: {
- "$ref" => (action == 'index' ? "#{k.to_s}List" : k.to_s)
+ "$ref" => (method_name == 'list' ? "#{k.to_s}List" : k.to_s)
},
scopes: [
"https://api.arvados.org/auth/arvados"
# We already built a generic method description, but we
# might find some more required parameters through
# introspection.
- method = d_methods[action.to_sym]
+ method = d_methods[method_key]
end
if ctl_class.respond_to? "_#{action}_requires_parameters".to_sym
ctl_class.send("_#{action}_requires_parameters".to_sym).each do |l, v|
end
end
end
- d_methods[action.to_sym] = method
-
- if action == 'index'
- list_method = method.dup
- list_method[:id].sub!('index', 'list')
- list_method[:description].sub!('Index', 'List')
- list_method[:description].sub!('index', 'list')
- d_methods[:list] = list_method
- end
+ d_methods[method_key] = method
end
end
end
api_accessible :user, extend: :common do |t|
t.add :owner_uuid
- t.add :user_id
t.add :api_client_id
# NB the "api_token" db column is a misnomer in that it's only the
# "secret" part of a token: a v1 token is just the secret, but a
# v2 token is "v2/uuid/secret".
t.add :api_token
t.add :created_by_ip_address
- t.add :default_owner_uuid
t.add :expires_at
t.add :last_used_at
t.add :last_used_by_ip_address
end
end
auth.update!(user: user,
- api_token: stored_secret,
- api_client_id: 0,
- scopes: scopes,
- expires_at: exp)
+ api_token: stored_secret,
+ api_client_id: 0,
+ scopes: scopes,
+ expires_at: exp)
Rails.logger.debug "cached remote token #{token_uuid} with secret #{stored_secret} and scopes #{scopes} in local db"
auth.api_token = secret
return auth
current_time = db_current_time
self.created_at ||= created_at_was || current_time
self.updated_at = current_time
- self.owner_uuid ||= current_default_owner if self.respond_to? :owner_uuid=
+ self.owner_uuid ||= current_user.uuid if current_user && self.respond_to?(:owner_uuid=)
if !anonymous_updater
self.modified_by_user_uuid = current_user ? current_user.uuid : nil
end
# Send welcome email
if send_notification_email.nil?
- send_notification_email = Rails.configuration.Mail.SendUserSetupNotificationEmail
+ send_notification_email = Rails.configuration.Users.SendUserSetupNotificationEmail
end
if newly_invited and send_notification_email and !Rails.configuration.Users.UserSetupMailText.empty?
arvcfg.declare_config "Containers.AlwaysUsePreemptibleInstances", Boolean, :preemptible_instances
arvcfg.declare_config "Containers.Logging.LogUpdatePeriod", ActiveSupport::Duration, :crunch_log_update_period
arvcfg.declare_config "Containers.Logging.LogUpdateSize", Integer, :crunch_log_update_size
-arvcfg.declare_config "Mail.MailchimpAPIKey", String, :mailchimp_api_key
-arvcfg.declare_config "Mail.MailchimpListID", String, :mailchimp_list_id
arvcfg.declare_config "Services.Controller.ExternalURL", URI
arvcfg.declare_config "Services.Workbench1.ExternalURL", URI, :workbench_address
arvcfg.declare_config "Services.Websocket.ExternalURL", URI, :websocket_address
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-# Config must be done before we files; otherwise they
-# won't be able to use Rails.configuration.* to initialize their
-# classes.
-
-require 'enable_jobs_api'
-
-Rails.application.configure do
- begin
- if ENV["ARVADOS_CONFIG"] != "none" && ActiveRecord::Base.connection.tables.include?('jobs')
- check_enable_legacy_jobs_api
- end
- rescue ActiveRecord::NoDatabaseError
- # Since rails 5.2, all initializers are run by rake tasks (like db:create),
- # see: https://github.com/rails/rails/issues/32870
- end
-end
Thread.current[:api_url_base]
end
- def current_default_owner
- # owner_uuid for newly created objects
- ((current_api_client_authorization &&
- current_api_client_authorization.default_owner_uuid) ||
- (current_user && current_user.default_owner_uuid) ||
- (current_user && current_user.uuid) ||
- nil)
- end
-
# Where is the client connecting from?
def current_api_client_ip_address
Thread.current[:api_client_ip_address]
+++ /dev/null
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-Disable_update_jobs_api_method_list = ConfigLoader.to_OrderedOptions({
- "jobs.create"=>{},
- "pipeline_instances.create"=>{},
- "pipeline_templates.create"=>{},
- "jobs.update"=>{},
- "pipeline_instances.update"=>{},
- "pipeline_templates.update"=>{},
- "job_tasks.create"=>{},
- "job_tasks.update"=>{}
- })
-
-Disable_jobs_api_method_list = ConfigLoader.to_OrderedOptions({
- "jobs.create"=>{},
- "pipeline_instances.create"=>{},
- "pipeline_templates.create"=>{},
- "jobs.get"=>{},
- "pipeline_instances.get"=>{},
- "pipeline_templates.get"=>{},
- "jobs.list"=>{},
- "pipeline_instances.list"=>{},
- "pipeline_templates.list"=>{},
- "jobs.index"=>{},
- "pipeline_instances.index"=>{},
- "pipeline_templates.index"=>{},
- "jobs.update"=>{},
- "pipeline_instances.update"=>{},
- "pipeline_templates.update"=>{},
- "jobs.queue"=>{},
- "jobs.queue_size"=>{},
- "job_tasks.create"=>{},
- "job_tasks.get"=>{},
- "job_tasks.list"=>{},
- "job_tasks.index"=>{},
- "job_tasks.update"=>{},
- "jobs.show"=>{},
- "pipeline_instances.show"=>{},
- "pipeline_templates.show"=>{},
- "job_tasks.show"=>{}})
-
-def check_enable_legacy_jobs_api
- # Create/update is permanently disabled (legacy functionality has been removed)
- Rails.configuration.API.DisabledAPIs.merge! Disable_update_jobs_api_method_list
-
- if Rails.configuration.Containers.JobsAPI.Enable == "false" ||
- (Rails.configuration.Containers.JobsAPI.Enable == "auto" &&
- ActiveRecord::Base.connection.select_value("SELECT COUNT(*) FROM jobs LIMIT 1") == 0)
- Rails.configuration.API.DisabledAPIs.merge! Disable_jobs_api_method_list
- end
-end
discovery_doc = JSON.parse(@response.body)
- group_index_params = discovery_doc['resources']['groups']['methods']['index']['parameters']
+ group_index_params = discovery_doc['resources']['groups']['methods']['list']['parameters']
group_contents_params = discovery_doc['resources']['groups']['methods']['contents']['parameters']
assert_equal group_contents_params.keys.sort, (group_index_params.keys + ['uuid', 'recursive', 'include', 'include_old_versions']).sort
discovery_doc = JSON.parse(@response.body)
- workflows_index_params = discovery_doc['resources']['workflows']['methods']['index']['parameters'] # no changes from super
- coll_index_params = discovery_doc['resources']['collections']['methods']['index']['parameters']
+ workflows_index_params = discovery_doc['resources']['workflows']['methods']['list']['parameters'] # no changes from super
+ coll_index_params = discovery_doc['resources']['collections']['methods']['list']['parameters']
assert_equal (workflows_index_params.keys + ['include_trash', 'include_old_versions']).sort, coll_index_params.keys.sort
token_count = get_token_count
# Test the POST scope.
post(v1_url('api_client_authorizations'),
- params: {api_client_authorization: {user_id: users(:active).id}},
+ params: {api_client_authorization: {owner_uuid: users(:active).uuid}},
headers: auth(:active_apitokens))
assert_response :success
assert_equal(token_count + 1, get_token_count,
group.update!(name: "test create and update name 1")
results = Group.where(uuid: group.uuid)
assert_equal "test create and update name 1", results.first.name, "Expected name to be updated to 1"
- updated_at_1 = results.first.updated_at.to_f
+ modified_at_1 = results.first.modified_at.to_f
# update 2
group.update!(name: "test create and update name 2")
results = Group.where(uuid: group.uuid)
assert_equal "test create and update name 2", results.first.name, "Expected name to be updated to 2"
- updated_at_2 = results.first.updated_at.to_f
+ modified_at_2 = results.first.modified_at.to_f
- assert_equal true, (updated_at_2 > updated_at_1), "Expected updated time 2 to be newer than 1"
+ assert_equal true, (modified_at_2 > modified_at_1), "Expected modified time 2 to be newer than 1"
end
test 'jsonb column' do
def insert_group uuid, owner_uuid, name, group_class
q = ActiveRecord::Base.connection.exec_query %{
-insert into groups (uuid, owner_uuid, name, group_class, created_at, updated_at)
+insert into groups (uuid, owner_uuid, name, group_class, created_at, updated_at, modified_at)
values ('#{uuid}', '#{owner_uuid}',
'#{name}', #{if group_class then "'"+group_class+"'" else 'NULL' end},
- statement_timestamp(), statement_timestamp())
+ statement_timestamp(), statement_timestamp(), statement_timestamp())
}
uuid
end
with self.assertRaises(SystemExit):
args = arvados_fuse.command.ArgumentParser().parse_args(['--version'])
- self.assertRegex(sys.stdout.getvalue(), "[0-9]+\.[0-9]+\.[0-9]+")
+ self.assertRegex(sys.stdout.getvalue(), r'[0-9]+\.[0-9]+\.[0-9]+')
sys.stderr.close()
sys.stderr = origerr
sys.stdout = origout
def setUp(self):
super(FuseMountTest, self).setUp()
- cw = arvados.CollectionWriter()
-
- cw.start_new_file('thing1.txt')
- cw.write("data 1")
- cw.start_new_file('thing2.txt')
- cw.write("data 2")
-
- cw.start_new_stream('dir1')
- cw.start_new_file('thing3.txt')
- cw.write("data 3")
- cw.start_new_file('thing4.txt')
- cw.write("data 4")
-
- cw.start_new_stream('dir2')
- cw.start_new_file('thing5.txt')
- cw.write("data 5")
- cw.start_new_file('thing6.txt')
- cw.write("data 6")
-
- cw.start_new_stream('dir2/dir3')
- cw.start_new_file('thing7.txt')
- cw.write("data 7")
-
- cw.start_new_file('thing8.txt')
- cw.write("data 8")
-
- cw.start_new_stream('edgecases')
- for f in ":/.../-/*/ ".split("/"):
- cw.start_new_file(f)
- cw.write('x')
-
- for f in ":/.../-/*/ ".split("/"):
- cw.start_new_stream('edgecases/dirs/' + f)
- cw.start_new_file('x/x')
- cw.write('x')
-
- self.testcollection = cw.finish()
- self.api.collections().create(body={"manifest_text":cw.manifest_text()}).execute()
+ cw = arvados.collection.Collection()
+ with cw.open('thing1.txt', 'w') as f:
+ f.write('data 1')
+ with cw.open('thing2.txt', 'w') as f:
+ f.write('data 2')
+
+ with cw.open('dir1/thing3.txt', 'w') as f:
+ f.write('data 3')
+ with cw.open('dir1/thing4.txt', 'w') as f:
+ f.write('data 4')
+
+ with cw.open('dir2/thing5.txt', 'w') as f:
+ f.write('data 5')
+ with cw.open('dir2/thing6.txt', 'w') as f:
+ f.write('data 6')
+
+ with cw.open('dir2/dir3/thing7.txt', 'w') as f:
+ f.write('data 7')
+ with cw.open('dir2/dir3/thing8.txt', 'w') as f:
+ f.write('data 8')
+
+ for fnm in ":/.../-/*/ ".split("/"):
+ with cw.open('edgecases/'+fnm, 'w') as f:
+ f.write('x')
+
+ for fnm in ":/.../-/*/ ".split("/"):
+ with cw.open('edgecases/dirs/'+fnm+'/x/x', 'w') as f:
+ f.write('x')
+
+ self.testcollection = cw.portable_data_hash()
+ self.test_manifest = cw.manifest_text()
+ self.api.collections().create(body={"manifest_text": self.test_manifest}).execute()
def runTest(self):
self.make_mount(fuse.CollectionDirectory, collection_record=self.testcollection)
self.collection_in_test_project = run_test_server.fixture('collections')['foo_collection_in_aproject']['name']
self.collection_in_filter_group = run_test_server.fixture('collections')['baz_file']['name']
- cw = arvados.CollectionWriter()
-
- cw.start_new_file('thing1.txt')
- cw.write("data 1")
+ cw = arvados.collection.Collection()
+ with cw.open('thing1.txt', 'w') as f:
+ f.write('data 1')
- self.testcollection = cw.finish()
+ self.testcollection = cw.portable_data_hash()
self.test_manifest = cw.manifest_text()
coll = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
self.test_manifest_pdh = coll['portable_data_hash']
def setUp(self, api=None):
super(FuseMagicTestPDHOnly, self).setUp(api=api)
- cw = arvados.CollectionWriter()
-
- cw.start_new_file('thing1.txt')
- cw.write("data 1")
+ cw = arvados.collection.Collection()
+ with cw.open('thing1.txt', 'w') as f:
+ f.write('data 1')
- self.testcollection = cw.finish()
+ self.testcollection = cw.portable_data_hash()
self.test_manifest = cw.manifest_text()
created = self.api.collections().create(body={"manifest_text":self.test_manifest}).execute()
self.testcollectionuuid = str(created['uuid'])
f.write('foo')
def checkContents(self):
- self.assertRegexpMatches(self.api.collections().get(uuid=self.testcoll['uuid']).execute()['manifest_text'], ' acbd18db') # md5(foo)
- self.assertRegexpMatches(self.api.collections().get(uuid=self.testcolleasy['uuid']).execute()['manifest_text'], ' f561aaf6') # md5(xxx)
+ self.assertRegex(self.api.collections().get(uuid=self.testcoll['uuid']).execute()['manifest_text'], r' acbd18db') # md5(foo)
+ self.assertRegex(self.api.collections().get(uuid=self.testcolleasy['uuid']).execute()['manifest_text'], r' f561aaf6') # md5(xxx)
@IntegrationTest.mount(argv=mnt_args)
@mock.patch('arvados.util.get_config_once')
self.testcollconflict = self.api.collections().create(body={"name": self.fusename}).execute()
get_config_once.return_value = {"Collections": {"ForwardSlashNameSubstitution": "[SLASH]"}}
self.pool_test(os.path.join(self.mnt, 'zzz'), self.fusename)
- self.assertRegexpMatches(self.api.collections().get(uuid=self.testcollconflict['uuid']).execute()['manifest_text'], ' acbd18db') # md5(foo)
+ self.assertRegex(self.api.collections().get(uuid=self.testcollconflict['uuid']).execute()['manifest_text'], r' acbd18db') # md5(foo)
# foo/bar/baz collection unchanged, because it is masked by foo[SLASH]bar[SLASH]baz
self.assertEqual(self.api.collections().get(uuid=self.testcoll['uuid']).execute()['manifest_text'], '')
@staticmethod
f.write("b2")
with open(os.path.join(tmpdir, "b1"), 'w') as f:
f.write("1b")
- self.assertRegex(current_manifest(tmpdir), "^\. ed4f3f67c70b02b29c50ce1ea26666bd\+4(\+\S+)? 0:2:b1 2:2:b2\n$")
+ self.assertRegex(current_manifest(tmpdir), r'^\. ed4f3f67c70b02b29c50ce1ea26666bd\+4(\+\S+)? 0:2:b1 2:2:b2\n$')
Containers: {},
InstanceTypes: {},
Login: {},
- Mail: { SupportEmailAddress: "arvados@example.com" },
RemoteClusters: {
"*": {
ActivateUsers: false,
StorageClasses: {
default: { Default: true, Priority: 0 },
},
- Users: {},
+ Users: { SupportEmailAddress: "arvados@example.com" },
Volumes: {},
Workbench: {},
},
Scheme: string
}
};
- Mail?: {
- SupportEmailAddress: string;
- };
Services: {
Controller: {
ExternalURL: string;
};
Users: {
AnonymousUserToken: string;
+ SupportEmailAddress: string;
};
}
},
Volumes: {},
Users: {
- AnonymousUserToken: ""
+ AnonymousUserToken: "",
+ SupportEmailAddress: "arvados@example.com",
},
...config,
});
createdAt: string;
updatedAt: string;
ownerUuid: string;
- defaultOwnerUuid: string;
scopes: string[];
}
export const getTokenV2 = (aca: ApiClientAuthorization): string =>
- `v2/${aca.uuid}/${aca.apiToken}`;
\ No newline at end of file
+ `v2/${aca.uuid}/${aca.apiToken}`;
export interface UserResource extends Resource, User {
kind: ResourceKind.USER;
- defaultOwnerUuid: string;
}
enum ApiClientAuthorizationsData {
API_CLIENT_AUTHORIZATION = 'api_client_authorization',
- DEFAULT_OWNER_UUID = 'default_owner_uuid'
+ EXPIRES_AT = 'expires_at'
}
enum LinkData {
data: dataApiClientAuthorization,
resourceKind: ApiClientAuthorizationsData.API_CLIENT_AUTHORIZATION,
resourcePrefix: ResourcePrefix.API_CLIENT_AUTHORIZATIONS,
- resourceKindProperty: ApiClientAuthorizationsData.DEFAULT_OWNER_UUID,
- property: dataApiClientAuthorization!.defaultOwnerUuid
+ resourceKindProperty: ApiClientAuthorizationsData.EXPIRES_AT,
+ property: dataApiClientAuthorization!.createdAt
});
dispatch<any>(initAdvancedTabDialog(advanceDataApiClientAuthorization));
break;
const userApiResponse = (apiResponse: UserResource): JSX.Element => {
const {
uuid, ownerUuid, createdAt, modifiedAt, modifiedByClientUuid, modifiedByUserUuid,
- email, firstName, lastName, username, isActive, isAdmin, prefs, defaultOwnerUuid,
+ email, firstName, lastName, username, isActive, isAdmin, prefs,
} = apiResponse;
const response = `
"uuid": "${uuid}",
"is_active": "${isActive},
"is_admin": "${isAdmin},
"prefs": "${stringifyObject(prefs)},
-"default_owner_uuid": "${defaultOwnerUuid},
"username": "${username}"`;
return <span style={{ marginLeft: '-15px' }}>{'{'} {response} {'\n'} <span style={{ marginLeft: '-15px' }}>{'}'}</span></span>;
const apiClientAuthorizationApiResponse = (apiResponse: ApiClientAuthorization): JSX.Element => {
const {
uuid, ownerUuid, apiToken, apiClientId, userId, createdByIpAddress, lastUsedByIpAddress,
- lastUsedAt, expiresAt, defaultOwnerUuid, scopes, updatedAt, createdAt
+ lastUsedAt, expiresAt, scopes, updatedAt, createdAt
} = apiResponse;
const response = `
"uuid": "${uuid}",
"owner_uuid": "${ownerUuid}",
"api_token": "${stringify(apiToken)}",
"api_client_id": "${stringify(apiClientId)}",
-"user_id": "${stringify(userId)}",
"created_by_ip_address": "${stringify(createdByIpAddress)}",
"last_used_by_ip_address": "${stringify(lastUsedByIpAddress)}",
"last_used_at": "${stringify(lastUsedAt)}",
"expires_at": "${stringify(expiresAt)}",
"created_at": "${stringify(createdAt)}",
"updated_at": "${stringify(updatedAt)}",
-"default_owner_uuid": "${stringify(defaultOwnerUuid)}",
"scopes": "${JSON.stringify(scopes, null, 2)}"`;
return <span style={{ marginLeft: '-15px' }}>{'{'} {response} {'\n'} <span style={{ marginLeft: '-15px' }}>{'}'}</span></span>;
<Grid item xs={7}>{data.apiClientAuthorization.apiToken}</Grid>
<Grid item xs={5}>Created by IP address</Grid>
<Grid item xs={7}>{data.apiClientAuthorization.createdByIpAddress || '(none)'}</Grid>
- <Grid item xs={5}>Default owner</Grid>
- <Grid item xs={7}>{data.apiClientAuthorization.defaultOwnerUuid || '(none)'}</Grid>
<Grid item xs={5}>Expires at</Grid>
<Grid item xs={7}>{formatDate(data.apiClientAuthorization.expiresAt) || '(none)'}</Grid>
<Grid item xs={5}>Last used at</Grid>
</Button>
</DialogActions>
</Dialog>
- );
\ No newline at end of file
+ );
export const TokenCreatedByIpAddress = withResourceData("createdByIpAddress", renderCommonDate);
-export const TokenDefaultOwnerUuid = withResourceData("defaultOwnerUuid", renderCommonData);
-
export const TokenExpiresAt = withResourceData("expiresAt", renderCommonDate);
export const TokenLastUsedAt = withResourceData("lastUsedAt", renderCommonDate);
import { DataExplorer } from 'views-components/data-explorer/data-explorer';
import { ResourcesState } from 'store/resources/resources';
import {
- CommonUuid, TokenApiClientId, TokenApiToken, TokenCreatedByIpAddress, TokenDefaultOwnerUuid, TokenExpiresAt,
+ CommonUuid, TokenApiClientId, TokenApiToken, TokenCreatedByIpAddress, TokenExpiresAt,
TokenLastUsedAt, TokenLastUsedByIpAddress, TokenScopes, TokenUserId
} from 'views-components/data-explorer/renderers';
import { ApiClientAuthorization } from 'models/api-client-authorization';
API_CLIENT_ID = 'API Client ID',
API_TOKEN = 'API Token',
CREATED_BY_IP_ADDRESS = 'Created by IP address',
- DEFAULT_OWNER_UUID = 'Default owner',
EXPIRES_AT = 'Expires at',
LAST_USED_AT = 'Last used at',
LAST_USED_BY_IP_ADDRESS = 'Last used by IP address',
filters: createTree(),
render: uuid => <TokenCreatedByIpAddress uuid={uuid} />
},
- {
- name: ApiClientAuthorizationPanelColumnNames.DEFAULT_OWNER_UUID,
- selected: true,
- configurable: true,
- filters: createTree(),
- render: uuid => <TokenDefaultOwnerUuid uuid={uuid} />
- },
{
name: ApiClientAuthorizationPanelColumnNames.EXPIRES_AT,
selected: true,
active: 'active',
},
clusterConfig: {
- Mail: {
+ Users: {
SupportEmailAddress: 'support@example.com'
}
} as ClusterConfigJSON,
it('should render component without email url when no email', () => {
// setup
- props.clusterConfig.Mail.SupportEmailAddress = '';
+ props.clusterConfig.Users.SupportEmailAddress = '';
// when
const wrapper = mount(
// and
expect(wrapper.find('a').length).toBe(1);
});
-});
\ No newline at end of file
+});
<p>
The page you requested was not found,
{
- !!clusterConfig.Mail && clusterConfig.Mail.SupportEmailAddress ?
- getEmailLink(clusterConfig.Mail.SupportEmailAddress, classes) :
+ !!clusterConfig.Users && clusterConfig.Users.SupportEmailAddress ?
+ getEmailLink(clusterConfig.Users.SupportEmailAddress, classes) :
'email us'
}
if you suspect this is a bug.
"""
def rerun_request(arv, container_requests_to_rerun, ct):
- requests = arvados.util.list_all(arv.container_requests().list, filters=[["container_uuid", "=", ct["uuid"]]])
+ requests = arvados.util.keyset_list_all(
+ arv.container_requests().list,
+ filters=[["container_uuid", "=", ct["uuid"]]],
+ order='uuid')
for cr in requests:
if cr["requesting_container_uuid"]:
rerun_request(arv, container_requests_to_rerun, arv.containers().get(uuid=cr["requesting_container_uuid"]).execute())
if (i % 100) == 0:
logging.log(lglvl, "%d/%d", i, len(busted_collections))
i += 1
- collections_to_delete = arvados.util.list_all(arv.collections().list, filters=[["portable_data_hash", "=", b]])
+ collections_to_delete = arvados.util.keyset_list_all(arv.collections().list, filters=[["portable_data_hash", "=", b]], order='uuid')
for d in collections_to_delete:
t = ""
if d["properties"].get("type") not in ("output", "log"):
ou = get_owner(arv, owners, d)
out.writerow((d["uuid"], "", d["name"], d["modified_at"], d["owner_uuid"], ou[0], ou[1], owners[ou[1]][0], t))
- maybe_containers_to_rerun = arvados.util.list_all(arv.containers().list, filters=[["output", "=", b]])
+ maybe_containers_to_rerun = arvados.util.keyset_list_all(arv.containers().list, filters=[["output", "=", b]], order='uuid')
for ct in maybe_containers_to_rerun:
rerun_request(arv, container_requests_to_rerun, ct)
for key_label in key_label_to_id_map:
logger.debug('Querying objects with property key "{}"'.format(key_label))
for resource in [arv.collections(), arv.groups()]:
- objs = arvados.util.list_all(
+ objs = arvados.util.keyset_list_all(
resource.list,
- order=['created_at'],
+ order='created_at',
select=['uuid', 'properties'],
filters=[['properties', 'exists', key_label]]
)
return 0
if __name__ == "__main__":
- sys.exit(main())
\ No newline at end of file
+ sys.exit(main())
projects / arvados.git / commitdiff