From: Peter Amstutz Date: Tue, 15 Nov 2016 17:29:14 +0000 (-0500) Subject: Merge branch '10346-rearrange-api-docs' closes #10346 X-Git-Tag: 1.1.0~603 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/eae48c31bb338689ec67fbc6a14a2e0b1fb5e3b6?hp=3af6db5dc4e2f08b2ebb49a82109c4325ad7fcc4 Merge branch '10346-rearrange-api-docs' closes #10346 --- diff --git a/doc/_config.yml b/doc/_config.yml index 5ed5af5cb5..96aea34d36 100644 --- a/doc/_config.yml +++ b/doc/_config.yml @@ -78,78 +78,67 @@ navbar: - sdk/index.html.textile.liquid - Python: - sdk/python/sdk-python.html.textile.liquid + - sdk/python/example.html.textile.liquid - sdk/python/python.html.textile.liquid - sdk/python/crunch-utility-libraries.html.textile.liquid - sdk/python/events.html.textile.liquid + - CLI: + - sdk/cli/install.html.textile.liquid + - sdk/cli/index.html.textile.liquid + - sdk/cli/reference.html.textile.liquid + - sdk/cli/subcommands.html.textile.liquid + - Go: + - sdk/go/index.html.textile.liquid + - sdk/go/example.html.textile.liquid - Perl: - sdk/perl/index.html.textile.liquid + - sdk/perl/example.html.textile.liquid - Ruby: - sdk/ruby/index.html.textile.liquid + - sdk/ruby/example.html.textile.liquid - Java: - sdk/java/index.html.textile.liquid - - Go: - - sdk/go/index.html.textile.liquid - - CLI: - - sdk/cli/index.html.textile.liquid - - sdk/cli/install.html.textile.liquid - - sdk/cli/reference.html.textile.liquid - - sdk/cli/subcommands.html.textile.liquid + - sdk/java/example.html.textile.liquid api: - Concepts: - api/index.html.textile.liquid - - api/authentication.html.textile.liquid + - api/tokens.html.textile.liquid + - api/requests.html.textile.liquid - api/methods.html.textile.liquid - api/resources.html.textile.liquid - - api/crunch-scripts.html.textile.liquid - api/permission-model.html.textile.liquid - - API Methods: + - api/storage.html.textile.liquid + - api/execution.html.textile.liquid + - Permission and authentication: - api/methods/api_client_authorizations.html.textile.liquid - api/methods/api_clients.html.textile.liquid - api/methods/authorized_keys.html.textile.liquid - - api/methods/collections.html.textile.liquid - - api/methods/container_requests.html.textile.liquid - - api/methods/containers.html.textile.liquid - api/methods/groups.html.textile.liquid - - api/methods/humans.html.textile.liquid - - api/methods/jobs.html.textile.liquid - - api/methods/job_tasks.html.textile.liquid - - api/methods/keep_disks.html.textile.liquid + - api/methods/users.html.textile.liquid + - System resources: - api/methods/keep_services.html.textile.liquid - api/methods/links.html.textile.liquid - api/methods/logs.html.textile.liquid - api/methods/nodes.html.textile.liquid + - api/methods/virtual_machines.html.textile.liquid + - api/methods/keep_disks.html.textile.liquid + - Data management: + - api/methods/collections.html.textile.liquid + - api/methods/repositories.html.textile.liquid + - Container engine: + - api/methods/container_requests.html.textile.liquid + - api/methods/containers.html.textile.liquid + - api/methods/workflows.html.textile.liquid + - Jobs engine (deprecated): + - api/crunch-scripts.html.textile.liquid + - api/methods/jobs.html.textile.liquid + - api/methods/job_tasks.html.textile.liquid - api/methods/pipeline_instances.html.textile.liquid - api/methods/pipeline_templates.html.textile.liquid - - api/methods/repositories.html.textile.liquid + - Metadata for bioinformatics: + - api/methods/humans.html.textile.liquid - api/methods/specimens.html.textile.liquid - api/methods/traits.html.textile.liquid - - api/methods/users.html.textile.liquid - - api/methods/virtual_machines.html.textile.liquid - - api/methods/workflows.html.textile.liquid - - Schema: - - api/schema/ApiClientAuthorization.html.textile.liquid - - api/schema/ApiClient.html.textile.liquid - - api/schema/AuthorizedKey.html.textile.liquid - - api/schema/Collection.html.textile.liquid - - api/schema/Container.html.textile.liquid - - api/schema/ContainerRequest.html.textile.liquid - - api/schema/Group.html.textile.liquid - - api/schema/Human.html.textile.liquid - - api/schema/Job.html.textile.liquid - - api/schema/JobTask.html.textile.liquid - - api/schema/KeepDisk.html.textile.liquid - - api/schema/KeepService.html.textile.liquid - - api/schema/Link.html.textile.liquid - - api/schema/Log.html.textile.liquid - - api/schema/Node.html.textile.liquid - - api/schema/PipelineInstance.html.textile.liquid - - api/schema/PipelineTemplate.html.textile.liquid - - api/schema/Repository.html.textile.liquid - - api/schema/Specimen.html.textile.liquid - - api/schema/Trait.html.textile.liquid - - api/schema/User.html.textile.liquid - - api/schema/VirtualMachine.html.textile.liquid - - api/schema/Workflow.html.textile.liquid installguide: - Overview: - install/index.html.textile.liquid @@ -168,12 +157,12 @@ navbar: - install/configure-azure-blob-storage.html.textile.liquid - install/install-keepproxy.html.textile.liquid - install/install-keep-web.html.textile.liquid - - Install Crunch v2 on SLURM: + - Containers API support on SLURM: - install/crunch2-slurm/install-prerequisites.html.textile.liquid - install/crunch2-slurm/install-compute-node.html.textile.liquid - install/crunch2-slurm/install-dispatch.html.textile.liquid - install/crunch2-slurm/install-test.html.textile.liquid - - Install Crunch v1: + - Jobs API support (deprecated): - install/install-crunch-dispatch.html.textile.liquid - install/install-compute-node.html.textile.liquid - Helpful hints: diff --git a/doc/_layouts/default.html.liquid b/doc/_layouts/default.html.liquid index ed232c6773..4c4e2542fa 100644 --- a/doc/_layouts/default.html.liquid +++ b/doc/_layouts/default.html.liquid @@ -2,7 +2,7 @@ - {% unless page.title == "Arvados | Documentation" %} Arvados | Documentation | {% endunless %}{{ page.title }} + {% unless page.title == "Arvados | Documentation" %} Arvados {% if page.navmenu %}| {{ page.navmenu }} {% endif %} | {% endunless %}{{ page.title }} diff --git a/doc/api/authentication.html.textile.liquid b/doc/api/authentication.html.textile.liquid deleted file mode 100644 index cbf75532ff..0000000000 --- a/doc/api/authentication.html.textile.liquid +++ /dev/null @@ -1,40 +0,0 @@ ---- -layout: default -navsection: api -navmenu: Concepts -title: Authentication - -... - - - -Every API request (except the authentication API itself) includes an @access_token@ parameter. - -table(table table-bordered table-condensed). -|Name|Type|Description| -|access_token|string|Access token returned by OAuth 2.0 authorization procedure| - -Many resources contain "actor" attributes like @modified_by@. An @access_token@ uniquely identifies a client (application or project) and an end-user. - -table(table table-bordered table-condensed). -|Name|Type|Description| -|modified_by_client_uuid|string|ID of API client| -|modified_by_user_uuid|string|ID of authenticated user| - -h2. Authorizing a client application - -The Arvados API uses the "OAuth 2.0 protocol":http://tools.ietf.org/html/draft-ietf-oauth-v2-22 for authentication and authorization. - -h3. Register your client application - -Before an application can run on an Arvados cloud, it needs to be registered with the cloud. - -That registration yields a @client_id@ and a @client_secret@. - -h3. Obtain an access code - -A client obtains an access code by means of a standard Oauth 2.0 flow. The access code is granted to it by an authorized user. The client requests one or more scopes, which translate to a set of requested permissions (reading, writing, etc). Unless the access is to be short-lived, a refresh token is also granted to the application. - -h3. Refresh the access code (optional) - -Access codes have a limited lifetime. A refresh token allows an application to request a new access token. diff --git a/doc/api/crunch-scripts.html.textile.liquid b/doc/api/crunch-scripts.html.textile.liquid index 98634cdd9b..4642ed540e 100644 --- a/doc/api/crunch-scripts.html.textile.liquid +++ b/doc/api/crunch-scripts.html.textile.liquid @@ -20,8 +20,8 @@ A task's context is provided in environment variables. table(table table-bordered table-condensed). |Environment variable|Description| -|@JOB_UUID@|UUID of the current "Job":schema/Job.html| -|@TASK_UUID@|UUID of the current "JobTask":schema/JobTask.html| +|@JOB_UUID@|UUID of the current "Job":methods/jobs.html| +|@TASK_UUID@|UUID of the current "JobTask":methods/job_tasks.html| |@ARVADOS_API_HOST@|Hostname and port number of API server| |@ARVADOS_API_TOKEN@|Authentication token to use with API calls made by the current task| @@ -43,4 +43,3 @@ task = arvados.api().job_tasks().get(uuid=os.environ['TASK_UUID']).execute() $sys.stderr.write("current task sequence number is %d" % task['sequence']) - diff --git a/doc/api/execution.html.textile.liquid b/doc/api/execution.html.textile.liquid new file mode 100644 index 0000000000..900e7659ef --- /dev/null +++ b/doc/api/execution.html.textile.liquid @@ -0,0 +1,27 @@ +--- +layout: default +navsection: api +title: Computing with Crunch +... + +Crunch is the name for the Arvados system for managing computation. It provides an abstract API to various clouds and HPC resource allocation and scheduling systems, and integrates closely with Keep storage and the Arvados permission system. + +h2. Container API + +Note: although the preferred API for Aravdos going forward, the Container API may not yet be available on all installations. + +# To submit work, create a "container request":{{site.baseurl}}/api/methods/container_requests.html in the @Committed@ state. +# The system will fufill the container request by creating or reusing a "Container object":{{site.baseurl}}/api/methods/containers.html and assigning it to the @container_uuid@ field. If the same request has been submitted in the past, it may reuse an existing container. The reuse behavior can be suppressed with @use_existing: false@ in the container request. +# The dispatcher process will notice a new container in @Queued@ state and submit a container executor to the underlying work queuing system (such as SLURM). +# The container executes. Upon termination the container goes into the @Complete@ state. If the container execution was interrupted or lost due to system failure, it will go into the @Cancelled@ state. +# When the container associated with the container request is completed, the container request will go into the @Final@ state. +# The @output_uuid@ field of the container request contains the uuid of output collection produced by container request. + +!{{site.baseurl}}/images/Crunch_dispatch.svg! + +h2. Job API (deprecated) + +# To submit work, create a "job":{{site.baseurl}}/api/methods/jobs.html . If the same job has been submitted in the past, it will return an existing job in @Completed@ state. +# The dispatcher process will notice a new job in @Queued@ state and attempt to allocate nodes to run the job. +# The job executes. +# Retrieve the @output@ field with the portable data hash of the collection with the output files of the job. diff --git a/doc/api/index.html.textile.liquid b/doc/api/index.html.textile.liquid index 81b2c1ce6a..d90a35efa4 100644 --- a/doc/api/index.html.textile.liquid +++ b/doc/api/index.html.textile.liquid @@ -5,46 +5,12 @@ title: API Reference ... +This reference describes the semantics of Arvados resources and how to programatically access Arvados via its REST API. Each resource listed in this section is exposed on the Arvados API server under the @/arvados/v1/@ path prefix, for example, @https://{{ site.arvados_api_host }}/arvados/v1/collections@. +h2. Discovery document -h2. Concepts +The API server publishes a machine-readable description of its endpoints and some additional site configuration values via a JSON-formatted discovery document. This is available at @/discovery/v1/apis/arvados/v1/rest@, for example @https://{{ site.arvados_api_host }}/discovery/v1/apis/arvados/v1/rest@. Some Arvados SDKs use the discovery document to generate language bindings. -* Each API uses the same "authentication mechanism":authentication.html. -* Resources in requests and responses adhere to a "common structure":resources.html. -* API transactions use common "REST methods":methods.html. -* API transactions are subject to a "permission model":permission-model.html. -* "Job tasks":schema/JobTask.html use some special API features. +h2. Workbench examples -h2. Resources - -h3. Generic Resources - -* "Collection":schema/Collection.html -* "Job":schema/Job.html -* "JobTask":schema/JobTask.html -* "Link":schema/Link.html -* "Log":schema/Log.html -* "PipelineTemplate":schema/PipelineTemplate.html -* "PipelineInstance":schema/PipelineInstance.html -* "Group":schema/Group.html -* "Human":schema/Human.html -* "Specimen":schema/Specimen.html -* "Trait":schema/Trait.html -* "User":schema/User.html - -h3. Authentication - -These Arvados resources govern authorization and "authentication":authentication.html: - -* "ApiClient":schema/ApiClient.html -* "ApiClientAuthorization":schema/ApiClientAuthorization.html -* "AuthorizedKey":schema/AuthorizedKey.html - -h3. Arvados Infrastructure - -These resources govern the Arvados infrastructure itself: Git repositories, Keep disks, active nodes, etc. - -* "KeepDisk":schema/KeepDisk.html -* "Node":schema/Node.html -* "Repository":schema/Repository.html -* "VirtualMachine":schema/VirtualMachine.html +Many Arvados Workbench pages, under the the *Advanced* tab, provide examples of API and SDK use for accessing the current resource . diff --git a/doc/api/methods.html.textile.liquid b/doc/api/methods.html.textile.liquid index 2d530d1473..e731cc7102 100644 --- a/doc/api/methods.html.textile.liquid +++ b/doc/api/methods.html.textile.liquid @@ -2,41 +2,73 @@ layout: default navsection: api navmenu: Concepts -title: REST methods +title: Common resource methods ... +The following methods are available for most resources. Some resources may limit who can perform certain operations. Consult documentation for individual resource types for details. +The methods are relative to the base URI, e.g. @/arvados/v1/resource_type@. For arguments specifying a *Location* of @path@, the value of the argument is incorporated into the path portion of the URI. For example, a @uuid@ of @aaaaa-bbbbb-ccccccccccccccc@ in a path position yields a URI of @/arvados/v1/resource_type/aaaaa-bbbbb-ccccccccccccccc@. -(using Group as an example) +Arguments specifying a *Location* of @query@ are incorporated into the query portion of the URI or request body. For example, @/arvados/v1/resource_type?resource_type={}@. -h2(#index). Index, list, search +h2. create -
-GET https://{{ site.arvados_api_host }}/arvados/v1/groups?filters=[["owner_uuid","=","xyzzy-tpzed-a4lcehql0dv2u25"]]
+The @create@ method creates a new object of the specified type.  Note that:
 
-POST https://{{ site.arvados_api_host }}/arvados/v1/groups
-_method=GET
-filters=[["owner_uuid","=","xyzzy-tpzed-a4lcehql0dv2u25"]]
-
+* Only the listed attributes (and "standard metadata":resources.html) are set +* Unset attributes will get default values +* The attributes of a given resource type are fixed (you cannot introduce new toplevel attributes) -→ Group resource list +This method corresponds to the HTTP request @POST /arvados/v1/resource_type@. A successful create call returns a copy of the new object. + +Arguments: + +table(table table-bordered table-condensed). +|_. Argument |_. Type |_. Description |_. Location | +|{resource_type}|object|Name is the singular form of the resource type, e.g. for the "collections" resource, this argument is "collection"|query|| + +h2. delete + +The @delete@ method deletes an object of the specified type. It corresponds to the HTTP request @DELETE /arvados/v1/resource_type/uuid@. A successful delete call returns a copy of the deleted object. + +Arguments: + +table(table table-bordered table-condensed). +|_. Argument |_. Type |_. Description |_. Location | +{background:#ccffcc}.|uuid|string|The UUID of the object in question.|path| + +h2. get + +The @get@ method gets a single object with the specified @uuid@. It corresponds to the HTTP request @GET /arvados/v1/resource_type/uuid@. + +Arguments: + +table(table table-bordered table-condensed). +|_. Argument |_. Type |_. Description |_. Location | +{background:#ccffcc}.|uuid|string|The UUID of the object in question.|path| + +h2(#index). list + +The @list@ method requests an list of resources of that type. It corresponds to the HTTP request @GET /arvados/v1/resource_type@. All resources support "list" method unless otherwise noted. + +Arguments: table(table table-bordered table-condensed). -|*Parameter name*|*Value*|*Description*| -|limit |integer|Maximum number of resources to return.| -|offset |integer|Skip the first 'offset' resources that match the given filter conditions.| -|filters |array |Conditions for selecting resources to return (see below).| +|_. Argument |_. Type |_. Description |_. Location | +|limit |integer|Maximum number of resources to return. If not provided, server will provide a default limit. Server may also impose a maximum number of records that can be returned in a single request.|query| +|offset |integer|Skip the first 'offset' number of resources that would be returned under the given filter conditions.|query| +|filters |array |"Conditions for selecting resources to return.":#filters|query| |order |array |Attributes to use as sort keys to determine the order resources are returned, each optionally followed by @asc@ or @desc@ to indicate ascending or descending order. Example: @["head_uuid asc","modified_at desc"]@ -Default: @["created_at desc"]@| +Default: @["created_at desc"]@|query| |select |array |Set of attributes to include in the response. Example: @["head_uuid","tail_uuid"]@ -Default: all available attributes, minus "manifest_text" in the case of collections.| +Default: all available attributes. As a special case, collections do not return "manifest_text" unless explicitly selected.|query| |distinct|boolean|@true@: (default) do not return duplicate objects -@false@: permitted to return duplicates| +@false@: permitted to return duplicates|query| -h3. Filters +h3(#filters). Available list method filters The value of the @filters@ parameter is an array of conditions. The @list@ method returns only the resources that satisfy all of the given conditions. In other words, the conjunction @AND@ is implicit. @@ -58,45 +90,23 @@ table(table table-bordered table-condensed). |@in@, @not in@|array of strings|@["script_version","in",["master","d00220fb38d4b85ca8fc28a8151702a2b9d1dec5"]]@| |@is_a@|string|@["head_uuid","is_a","arvados#pipelineInstance"]@| -h2. Create - -
-POST https://{{ site.arvados_api_host }}/arvados/v1/groups
-group={"name":"fresh new group"}
-
- -→ Group resource - -h2. Delete - -
-DELETE https://{{ site.arvados_api_host }}/arvados/v1/groups/xyzzy-ldvyl-vyydjeplwaa6emg
-
+h3. Results of list method -→ Group resource +A successful call to list will return the following object. -h2. Update - -
-PUT https://{{ site.arvados_api_host }}/arvados/v1/groups/xyzzy-ldvyl-vyydjeplwaa6emg
-group={"uuid":"xyzzy-ldvyl-vyydjeplwaa6emg", "name":"Important group"}
-
- -→ Group resource - -
-PUT https://{{ site.arvados_api_host }}/arvados/v1/groups/xyzzy-ldvyl-vyydjeplwaa6emg
-group[uuid]=xyzzy-ldvyl-vyydjeplwaa6emg
-group[name]=Important group
-
- -→ Group resource +table(table table-bordered table-condensed). +|_. Attribute |_. Type |_. Description | +|kind|string|type of objects returned| +|offset|integer|query offset in effect| +|limit|integer|query limit in effect| +|items|array|actual query payload, an array of resource objects| +|items_available|integer|total items available matching query| -More appropriate (but not yet implemented): +h2. update -
-PATCH https://{{ site.arvados_api_host }}/arvados/v1/groups/xyzzy-ldvyl-vyydjeplwaa6emg
-group={"uuid":"xyzzy-ldvyl-vyydjeplwaa6emg", "name":"Important group"}
-
+The @update@ method updates fields on the object with the specified @uuid@. It corresponds to the HTTP request @PUT /arvados/v1/resource_type/uuid@. Note that only the listed attributes (and "standard metadata":resources.html) are updated, unset attributes will retain their previous values, and the attributes of a given resource type are fixed (you cannot introduce new toplevel attributes). Also note that updates replace the value of the attribute, so if an attribute has an object value, the entire object is replaced. A successful update call returns the updated copy of the object. -→ Group resource +table(table table-bordered table-condensed). +|_. Argument |_. Type |_. Description |_. Location | +{background:#ccffcc}.|uuid|string|The UUID of the resource in question.|path|| +|{resource_type}|object||query|| diff --git a/doc/api/methods/api_client_authorizations.html.textile.liquid b/doc/api/methods/api_client_authorizations.html.textile.liquid index 7af9711b9d..ab49610ab1 100644 --- a/doc/api/methods/api_client_authorizations.html.textile.liquid +++ b/doc/api/methods/api_client_authorizations.html.textile.liquid @@ -6,24 +6,50 @@ title: "api_client_authorizations" ... -See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html - API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/api_client_authorizations@ -Required arguments are displayed in %{background:#ccffcc}green%. +Object type: @gj3su@ + +Example UUID: @zzzzz-gj3su-0123456789abcde@ + +h2. Resource + +The @api_client_authorizations@ resource stores the API tokens that have been issued to permit access the API server. + +An ApiClientAuthorization is *not* a generic Arvados resource. The full list of properties that belong to an ApiClientAuthorization is: + +table(table table-bordered table-condensed). +|_. Attribute|_. Type|_. Description|_. Example| +|uuid|string|An identifier used to refer to the token without exposing the actual token.|| +|api_token|string|The actual token string that is expected in the Authorization header.|| +|api_client_id|integer|-|| +|user_id|integer|-|| +|created_by_ip_address|string|-|| +|last_used_by_ip_address|string|The network address of the most recent client using this token.|| +|last_used_at|datetime|Timestamp of the most recent request using this token.|| +|expires_at|datetime|Time at which the token is no longer valid. May be set to a time in the past in order to immediately expire a token.|| +|owner_uuid|string|The user associated with the token. All operations using this token are checked against the permissions of this user.|| +|scopes|array|A list of resources this token is allowed to access. A scope of ["all"] allows all resources. See "API Authorization":{{site.baseurl}}/api/tokens.html#scopes for details.|| + +h2. Methods + +See "Common resource methods":{{site.baseurl}}/api/methods.html for more information about @create@, @delete@, @get@, @list@, and @update@. +Required arguments are displayed in %{background:#ccffcc}green%. -h2. create +h3(#create). create Create a new ApiClientAuthorization. +Regular users may only create self-owned API tokens, but may provide a restricted "scope":{{site.baseurl}}/api/tokens.html#scopes . Administrators may create API tokens corresponding to any user. + Arguments: table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | |api_client_authorization|object||query|| -h2. create_system_auth +h3. create_system_auth create_system_auth api_client_authorizations @@ -34,7 +60,7 @@ table(table table-bordered table-condensed). |api_client_id|integer||query|| |scopes|array||query|| -h2. delete +h3. delete Delete an existing ApiClientAuthorization. @@ -44,9 +70,9 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the ApiClientAuthorization in question.|path|| -h2. get +h3. get -Gets a ApiClientAuthorization's metadata by UUID. +Gets an ApiClientAuthorization's metadata by UUID. Arguments: @@ -54,19 +80,13 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the ApiClientAuthorization in question.|path|| -h2. list +h3. list List api_client_authorizations. -Arguments: - -table(table table-bordered table-condensed). -|_. Argument |_. Type |_. Description |_. Location |_. Example | -|limit|integer (default 100)|Maximum number of api_client_authorizations to return.|query|| -|order|string|Order in which to return matching api_client_authorizations.|query|| -|filters|array|Conditions for filtering api_client_authorizations.|query|| +See "common resource list method.":{{site.baseurl}}/api/methods.html#index -h2. update +h3. update Update attributes of an existing ApiClientAuthorization. diff --git a/doc/api/methods/api_clients.html.textile.liquid b/doc/api/methods/api_clients.html.textile.liquid index 056cc30812..f0224fbc48 100644 --- a/doc/api/methods/api_clients.html.textile.liquid +++ b/doc/api/methods/api_clients.html.textile.liquid @@ -6,13 +6,31 @@ title: "api_clients" ... -See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html - API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/api_clients@ +Object type: @ozdt8@ + +Example UUID: @zzzzz-ozdt8-0123456789abcde@ + +h2. Resource + +The "api_clients" resource determines if web applications that have gone through the browser login flow may create or list API tokens. + +Each ApiClient has, in addition to the "Common resource fields":{{site.baseurl}}/api/resources.html: + +table(table table-bordered table-condensed). +|_. Attribute|_. Type|_. Description|_. Example| +|name|string||| +|url_prefix|string||| +|is_trusted|boolean|Trusted by users to handle their API tokens (ApiClientAuthorizations).|| + +h2. Methods + +See "Common resource methods":{{site.baseurl}}/api/methods.html for more information about @create@, @delete@, @get@, @list@, and @update@. + Required arguments are displayed in %{background:#ccffcc}green%. -h2. create +h3. create Create a new ApiClient. @@ -22,7 +40,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | |api_client|object||query|| -h2. delete +h3. delete Delete an existing ApiClient. @@ -32,7 +50,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the ApiClient in question.|path|| -h2. get +h3. get Gets a ApiClient's metadata by UUID. @@ -42,19 +60,13 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the ApiClient in question.|path|| -h2. list +h3. list List api_clients. -Arguments: - -table(table table-bordered table-condensed). -|_. Argument |_. Type |_. Description |_. Location |_. Example | -|limit|integer (default 100)|Maximum number of api_clients to return.|query|| -|order|string|Order in which to return matching api_clients.|query|| -|filters|array|Conditions for filtering api_clients.|query|| +See "common resource list method.":{{site.baseurl}}/api/methods.html#index -h2. update +h3. update Update attributes of an existing ApiClient. diff --git a/doc/api/methods/authorized_keys.html.textile.liquid b/doc/api/methods/authorized_keys.html.textile.liquid index 9727c5730c..2912ba83d1 100644 --- a/doc/api/methods/authorized_keys.html.textile.liquid +++ b/doc/api/methods/authorized_keys.html.textile.liquid @@ -6,14 +6,33 @@ title: "authorized_keys" ... -See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html - API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/authorized_keys@ -Required arguments are displayed in %{background:#ccffcc}green%. +Object type: @fngyi@ + +Example UUID: @zzzzz-fngyi-0123456789abcde@ + +h2. Resource + +The authorized_keys resource stores SSH public keys which grant access to virtual machines or git repositories on the Arvados cluster as the user in @authorized_user_uuid@. + +Each AuthorizedKey has, in addition to the "Common resource fields":{{site.baseurl}}/api/resources.html: + +table(table table-bordered table-condensed). +|_. Attribute|_. Type|_. Description|_. Example| +|name|string|A name to help the user manage their keys.|| +|key_type|string|Public key type, currently only supports "SSH"|| +|authorized_user_uuid|string|The user to which this key belongs. Authentication using this key authenticates as this user.|| +|public_key|text|The actual public key material, e.g. from @~/.ssh/id_rsa.pub@|| +|expires_at|datetime|Expiration date after which the key is no longer valid.|| +h2. Methods -h2. create +See "Common resource methods":{{site.baseurl}}/api/methods.html for more information about @create@, @delete@, @get@, @list@, and @update@. + +Required arguments are displayed in %{background:#ccffcc}green%. + +h3. create Create a new AuthorizedKey. @@ -23,7 +42,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | |authorized_key|object||query|| -h2. delete +h3. delete Delete an existing AuthorizedKey. @@ -33,7 +52,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the AuthorizedKey in question.|path|| -h2. get +h3. get Gets a AuthorizedKey's metadata by UUID. @@ -43,19 +62,13 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the AuthorizedKey in question.|path|| -h2. list +h3. list List authorized_keys. -Arguments: - -table(table table-bordered table-condensed). -|_. Argument |_. Type |_. Description |_. Location |_. Example | -|limit|integer (default 100)|Maximum number of authorized_keys to return.|query|| -|order|string|Order in which to return matching authorized_keys.|query|| -|filters|array|Conditions for filtering authorized_keys.|query|| +See "common resource list method.":{{site.baseurl}}/api/methods.html#index -h2. update +h3. update Update attributes of an existing AuthorizedKey. diff --git a/doc/api/methods/collections.html.textile.liquid b/doc/api/methods/collections.html.textile.liquid index f5e685e2e9..808125a9e6 100644 --- a/doc/api/methods/collections.html.textile.liquid +++ b/doc/api/methods/collections.html.textile.liquid @@ -6,13 +6,45 @@ title: "collections" ... -See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html - API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/collections@ +Object type: @4zz18@ + +Example UUID: @zzzzz-4zz18-0123456789abcde@ + +h2. Resource + +Collections describe sets of files in terms of data blocks stored in Keep. See "storage in Keep":{{site.baseurl}}/api/storage.html for details. + +Each collection has, in addition to the "Common resource fields":{{site.baseurl}}/api/resources.html: + +table(table table-bordered table-condensed). +|_. Attribute|_. Type|_. Description|_. Example| +|name|string||| +|description|text||| +|portable_data_hash|string|The MD5 sum of the manifest text stripped of block hints other than the size hint.|| +|manifest_text|text||| +|replication_desired|number|Minimum storage replication level desired for each data block referenced by this collection. A value of @null@ signifies that the site default replication level (typically 2) is desired.|@2@| +|replication_confirmed|number|Replication level most recently confirmed by the storage system. This field is null when a collection is first created, and is reset to null when the manifest_text changes in a way that introduces a new data block. An integer value indicates the replication level of the _least replicated_ data block in the collection.|@2@, null| +|replication_confirmed_at|datetime|When replication_confirmed was confirmed. If replication_confirmed is null, this field is also null.|| + +h3. Conditions of creating a Collection + +The @portable_data_hash@ and @manifest_text@ attributes must be provided when creating a Collection. The cryptographic digest of the supplied @manifest_text@ must match the supplied @portable_data_hash@. + +h3. Side effects of creating a Collection + +Referenced blocks are protected from garbage collection in Keep. + +Data can be shared with other users via the Arvados permission model. + +h2. Methods + +See "Common resource methods":{{site.baseurl}}/api/methods.html for more information about @create@, @delete@, @get@, @list@, and @update@. + Required arguments are displayed in %{background:#ccffcc}green%. -h2. create +h3. create Create a new Collection. @@ -22,7 +54,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | |collection|object||query|| -h2. delete +h3. delete Delete an existing Collection. @@ -32,7 +64,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the Collection in question.|path|| -h2. get +h3. get Gets a Collection's metadata by UUID. @@ -42,22 +74,15 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the Collection in question.|path|| -h2. list +h3. list List collections. -Arguments: - -table(table table-bordered table-condensed). -|_. Argument |_. Type |_. Description |_. Location |_. Example | -|limit|integer (default 100)|Maximum number of collections to return.|query|| -|order|string|Order in which to return matching collections.|query|| -|filters|array|Conditions for filtering collections.|query|| -|select|array|Data fields to return in the result list.|query|@["uuid", "manifest_text"]@| +See "common resource list method.":{{site.baseurl}}/api/methods.html#index Note: Because adding access tokens to manifests can be computationally expensive, the @manifest_text@ field is not included in results by default. If you need it, pass a @select@ parameter that includes @manifest_text@. -h2. update +h3. update Update attributes of an existing Collection. diff --git a/doc/api/methods/container_requests.html.textile.liquid b/doc/api/methods/container_requests.html.textile.liquid index 2603079560..304226d5de 100644 --- a/doc/api/methods/container_requests.html.textile.liquid +++ b/doc/api/methods/container_requests.html.textile.liquid @@ -6,70 +6,124 @@ title: "container_requests" ... -See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html - API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/container_requests@ +Object type: @xvhdp@ + +Example UUID: @zzzzz-xvhdp-0123456789abcde@ + +h2. Resource + +A container request is a request for the Arvados cluster to perform some computational work. See "computing with Crunch":{{site.baseurl}}/api/execution.html for details. + +Each ContainerRequest offers the following attributes, in addition to the "Common resource fields":{{site.baseurl}}/api/resources.html: + +All attributes are optional, unless otherwise marked as required. + +table(table table-bordered table-condensed). +|_. Attribute|_. Type|_. Description|_. Notes| +|name|string|The name of the container_request.|| +|description|string|The description of the container_request.|| +|properties|hash|Client-defined structured data that does not affect how the container is run.|| +|state|string|The allowed states are "Uncommitted", "Committed", and "Final".|Once a request is Committed, the only attributes that can be modified are priority, container_uuid, and container_count_max. A request in the "Final" state cannot have any of its functional parts modified (i.e., only name, description, and properties fields can be modified).| +|requesting_container_uuid|string|The uuid of the parent container that created this container_request, if any. Represents a process tree.|The priority of this container_request is inherited from the parent container, if the parent container is cancelled, this container_request will be cancelled as well.| +|container_uuid|string|The uuid of the container that satisfies this container_request. The system may return a preexisting Container that matches the container request criteria. See "Container reuse":#container_reuse for more details.|Container reuse is the default behavior, but may be disabled with @use_existing: false@ to always create a new container.| +|container_count_max|integer|Maximum number of containers to start, i.e., the maximum number of "attempts" to be made.|| +|mounts|hash|Objects to attach to the container's filesystem and stdin/stdout.|See "Mount types":#mount_types for more details.| +|runtime_constraints|hash|Restrict the container's access to compute resources and the outside world.|Required when in "Committed" state. e.g.,
{
+  "ram":12000000000,
+  "vcpus":2,
+  "API":true
+}
See "Runtime constraints":#runtime_constraints for more details.| +|container_image|string|Portable data hash of a collection containing the docker image to run the container.|Required.| +|environment|hash|Environment variables and values that should be set in the container environment (@docker run --env@). This augments and (when conflicts exist) overrides environment variables given in the image's Dockerfile.|| +|cwd|string|Initial working directory, given as an absolute path (in the container) or a path relative to the WORKDIR given in the image's Dockerfile.|Required.| +|command|array of strings|Command to execute in the container.|Required. e.g., @["echo","hello"]@| +|output_path|string|Path to a directory or file inside the container that should be preserved as container's output when it finishes. This path must be, or be inside, one of the mount targets. For best performance, point output_path to a writable collection mount.|Required.| +|priority|integer|Higher value means spend more resources on this container_request, i.e., go ahead of other queued containers, bring up more nodes etc.|Priority 0 means a container should not be run on behalf of this request. Clients are expected to submit container requests with zero priority in order to prevew the container that will be used to satisfy it. Priority can be null if and only if state!="Committed".| +|expires_at|datetime|After this time, priority is considered to be zero.|Not yet implemented.| +|use_existing|boolean|If possible, use an existing (non-failed) container to satisfy the request instead of creating a new one.|Default is true| +|filters|string|Additional constraints for satisfying the container_request, given in the same form as the filters parameter accepted by the container_requests.list API.| + +h2(#mount_types). {% include 'mount_types' %} + +h2(#runtime_constraints). {% include 'container_runtime_constraints' %} + +h2(#container_reuse). Container reuse + +When a container request is "Committed", the system will try to find and reuse any preexisting Container with the same exact command, cwd, environment, output_path, container_image, mounts, and runtime_constraints as this container request. The serialized fields environment, mounts and runtime_constraints are sorted to facilitate comparison. + +The system will use the following scheme to determine which Container to consider for reuse: A Container with the same exact command, cwd, environment, output_path, container_image, mounts, and runtime_constraints as this container request and, +* The oldest successfully finished container, i.e., in state "Complete" with exit_code of 0. If matching containers with different outputs are found, the system will forgo reusing any of these finished containers and instead look for suitable containers in other states +* The oldest "Running" container with the highest progress, i.e., the container that is most likely to finish first +* The oldest "Locked" container with the highest priority, i.e., the container that is most likely to start first +* The oldest "Queued" container with the highest priority, i.e, the container that is most likely to start first + +h2(#cancel_container). Canceling a container request + +A container request may be canceled by setting it's priority to 0, using an update call. + +When a container request is canceled, it will still reflect the state of the Container it is associated with via the container_uuid attribute. If that Container is being reused by any other container_requests that are still active, i.e., not yet canceled, that Container may continue to run or be scheduled to run by the system in future. However, if no other container_requests are using that Contianer, then the Container will get canceled as well. + +h2. Methods + +See "Common resource methods":{{site.baseurl}}/api/methods.html for more information about @create@, @delete@, @get@, @list@, and @update@. + Required arguments are displayed in %{background:#ccffcc}green%. h2(#create). create -Create a new ContainerRequest. +Create a new container request. Arguments: table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | -{background:#ccffcc}.|container_request|object|See "ContainerRequest resource":{{site.baseurl}}/api/schema/ContainerRequest.html|request body|| +{background:#ccffcc}.|container_request|object|Container request resource.|request body|| + The request body must include the required attributes command, container_image, cwd, and output_path. It can also inlcude other attributes such as environment, mounts, and runtime_constraints. -h2. delete +h3. delete -Delete an existing ContainerRequest. +Delete an existing container request. Arguments: table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | -{background:#ccffcc}.|uuid|string|The UUID of the ContainerRequest in question.|path|| +{background:#ccffcc}.|uuid|string|The UUID of the container request in question.|path|| -h2. get +h3. get -Get a ContainerRequest's metadata by UUID. +Get a container request's metadata by UUID. Arguments: table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | -{background:#ccffcc}.|uuid|string|The UUID of the ContainerRequest in question.|path|| +{background:#ccffcc}.|uuid|string|The UUID of the container request in question.|path|| -h2. list +h3. list List container_requests. -Arguments: - -table(table table-bordered table-condensed). -|_. Argument |_. Type |_. Description |_. Location |_. Example | -|limit|integer (default 100)|Maximum number of container_requests to return.|query|| -|order|string|Order in which to return matching container_requests.|query|| -|filters|array|Conditions for filtering container_requests.|query|| +See "common resource list method.":{{site.baseurl}}/api/methods.html#index -See the create method documentation for more information about ContainerRequest-specific filters. +See the create method documentation for more information about container request-specific filters. -h2. update +h3. update -Update attributes of an existing ContainerRequest. +Update attributes of an existing container request. Arguments: table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | -{background:#ccffcc}.|uuid|string|The UUID of the ContainerRequest in question.|path|| +{background:#ccffcc}.|uuid|string|The UUID of the container request in question.|path|| |container_request|object||query|| {% include 'notebox_begin' %} Setting the priority of a committed container_request to 0 may cancel a running container assigned for it. -See "Canceling a ContainerRequest":{{site.baseurl}}/api/schema/ContainerRequest.html#cancel_container for further details. +See "Canceling a container request":{{site.baseurl}}/api/methods/container_requests.html#cancel_container for further details. {% include 'notebox_end' %} diff --git a/doc/api/methods/containers.html.textile.liquid b/doc/api/methods/containers.html.textile.liquid index c39b0926c1..221141cebc 100644 --- a/doc/api/methods/containers.html.textile.liquid +++ b/doc/api/methods/containers.html.textile.liquid @@ -6,10 +6,62 @@ title: "containers" ... -See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html - API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/containers@ +Object type: @dz642@ + +Example UUID: @zzzzz-dz642-0123456789abcde@ + +h2. Resource + +A container is work order to be dispatched to an Arvados cluster to perform some computational work. A container is created in response to a container request. See "computing with Crunch":{{site.baseurl}}/api/execution.html for details. + +Each Container offers the following attributes, in addition to the "Common resource fields":{{site.baseurl}}/api/resources.html: + +table(table table-bordered table-condensed). +|_. Attribute|_. Type|_. Description|_. Notes| +|state|string|The allowed states are "Queued", "Locked", "Running", "Cancelled" and "Complete".|See "Container states":#container_states for more details.| +|started_at|datetime|When this container started running.|Null if container has not yet started.| +|finished_at|datetime|When this container finished.|Null if container has not yet finished.| +|log|string|Portable data hash of the collection containing logs from a completed container run.|Null if the container is not yet finished.| +|environment|hash|Environment variables and values that should be set in the container environment (@docker run --env@). This augments and (when conflicts exist) overrides environment variables given in the image's Dockerfile.|Must be equal to a ContainerRequest's environment in order to satisfy the ContainerRequest.| +|cwd|string|Initial working directory.|Must be equal to a ContainerRequest's cwd in order to satisfy the ContainerRequest| +|command|array of strings|Command to execute.| Must be equal to a ContainerRequest's command in order to satisfy the ContainerRequest.| +|output_path|string|Path to a directory or file inside the container that should be preserved as this container's output when it finishes.|Must be equal to a ContainerRequest's output_path in order to satisfy the ContainerRequest.| +|mounts|hash|Must contain the same keys as the ContainerRequest being satisfied. Each value must be within the range of values described in the ContainerRequest at the time the Container is assigned to the ContainerRequest.|See "Mount types":#mount_types for more details.| +|runtime_constraints|hash|Compute resources, and access to the outside world, that are / were available to the container. +Generally this will contain additional keys that are not present in any corresponding ContainerRequests: for example, even if no ContainerRequests specified constraints on the number of CPU cores, the number of cores actually used will be recorded here.|e.g., +
{
+  "ram":12000000000,
+  "vcpus":2,
+  "API":true
+}
See "Runtime constraints":#runtime_constraints for more details.| +|output|string|Portable data hash of the output collection.|Null if the container is not yet finished.| +|container_image|string|Portable data hash of a collection containing the docker image used to run the container.|| +|progress|number|A number between 0.0 and 1.0 describing the fraction of work done.|| +|priority|integer|Priority assigned by the system, taking into account the priorities of all associated ContainerRequests.|| +|exit_code|integer|Process exit code.|Null if state!="Complete"| +|auth_uuid|string|UUID of a token to be passed into the container itself, used to access Keep-backed mounts, etc.|Null if state∉{"Locked","Running"}| +|locked_by_uuid|string|UUID of a token, indicating which dispatch process changed state to Locked. If null, any token can be used to lock. If not null, only the indicated token can modify this container.|Null if state∉{"Locked","Running"}| + +h2(#container_states). Container states + +table(table table-bordered table-condensed). +|_. State|_. Sgnificance|_. Allowed next| +|Queued|Waiting for a dispatcher to lock it and try to run the container.|Locked, Cancelled| +|Locked|A dispatcher has "taken" the container and is allocating resources for it. The container has not started yet.|Queued, Running, Cancelled| +|Running|Resources have been allocated and the contained process has been started (or is about to start). Crunch-run _must_ set state to Running _before_ there is any possibility that user code will run in the container.|Complete, Cancelled| +|Complete|Container was running, and the contained process/command has exited.|-| +|Cancelled|The container did not run long enough to produce an exit code. This includes cases where the container didn't even start, cases where the container was interrupted/killed before it exited by itself (e.g., priority changed to 0), and cases where some problem prevented the system from capturing the contained process's exit status (exit code and output).|-| + +h2(#mount_types). {% include 'mount_types' %} + +h2(#runtime_constraints). {% include 'container_runtime_constraints' %} + +h2. Methods + +See "Common resource methods":{{site.baseurl}}/api/methods.html for more information about @create@, @delete@, @get@, @list@, and @update@. + Required arguments are displayed in %{background:#ccffcc}green%. h2(#create). create @@ -20,9 +72,9 @@ Arguments: table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | -{background:#ccffcc}.|container|object|See "Container resource":{{site.baseurl}}/api/schema/Container.html|request body|| +{background:#ccffcc}.|container|object|Container resource|request body|| -h2. delete +h3. delete Delete an existing Container. @@ -32,7 +84,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the Container in question.|path|| -h2. get +h3. get Get a Container's metadata by UUID. @@ -42,21 +94,15 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the Container in question.|path|| -h2. list +h3. list List containers. -Arguments: - -table(table table-bordered table-condensed). -|_. Argument |_. Type |_. Description |_. Location |_. Example | -|limit|integer (default 100)|Maximum number of containers to return.|query|| -|order|string|Order in which to return matching containers.|query|| -|filters|array|Conditions for filtering containers.|query|| +See "common resource list method.":{{site.baseurl}}/api/methods.html#index See the create method documentation for more information about Container-specific filters. -h2. update +h3. update Update attributes of an existing Container. @@ -67,7 +113,7 @@ table(table table-bordered table-condensed). {background:#ccffcc}.|uuid|string|The UUID of the Container in question.|path|| |container|object||query|| -h2. auth +h3. auth Get the api_client_authorization record indicated by this container's auth_uuid, which belongs to the container's locked_by_uuid. diff --git a/doc/api/methods/groups.html.textile.liquid b/doc/api/methods/groups.html.textile.liquid index cd9633db42..7a15d20d5a 100644 --- a/doc/api/methods/groups.html.textile.liquid +++ b/doc/api/methods/groups.html.textile.liquid @@ -3,18 +3,35 @@ layout: default navsection: api navmenu: API Methods title: "groups" - ... +API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/groups@ -See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html +Object type: @j7d0g@ -API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/groups@ +Example UUID: @zzzzz-j7d0g-0123456789abcde@ -Required arguments are displayed in %{background:#ccffcc}green%. +h2. Resource + +Groups provides a way to apply the same permissions to a set of Arvados objects. See "permission model":{{site.baseurl}}/api/permission-model.html for details. + +Each Group has, in addition to the "Common resource fields":{{site.baseurl}}/api/resources.html: + +table(table table-bordered table-condensed). +|_. Attribute|_. Type|_. Description|_. Example| +|name|string||| +|group_class|string|Type of group. This does not affect behavior, but determines how the group is presented in the user interface. For example, @project@ indicates that the group should be displayed by Workbench and arv-mount as a project for organizing and naming objects.|@"project"@ +null| +|description|text||| +|writable_by|array|List of UUID strings identifying Users and other Groups that have write permission for this Group. Only users who are allowed to administer the Group will receive a full list. Other users will receive a partial list that includes the Group's owner_uuid and (if applicable) their own user UUID.|| +h2. Methods -h2. contents +See "Common resource methods":{{site.baseurl}}/api/methods.html for more information about @create@, @delete@, @get@, @list@, and @update@. + +Required arguments are displayed in %{background:#ccffcc}green%. + +h3. contents Retrieve a list of items owned by the group. @@ -31,7 +48,7 @@ Note: Because adding access tokens to manifests can be computationally expensive Note: Use filters with the attribute format @.@ to filter items of a specific type. For example: @["pipeline_instances.state", "=", "Complete"]@ to filter @pipeline_instances@ where @state@ is @Complete@. All other types of items owned by this group will be unimpacted by this filter and will still be included. -h2. create +h3. create Create a new Group. @@ -41,7 +58,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | |group|object||query|| -h2. delete +h3. delete Delete an existing Group. @@ -51,7 +68,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the Group in question.|path|| -h2. get +h3. get Gets a Group's metadata by UUID. @@ -61,19 +78,13 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the Group in question.|path|| -h2. list +h3. list List groups. -Arguments: - -table(table table-bordered table-condensed). -|_. Argument |_. Type |_. Description |_. Location |_. Example | -|limit|integer (default 100)|Maximum number of groups to return.|query|| -|order|string|Order in which to return matching groups.|query|| -|filters|array|Conditions for filtering groups.|query|| +See "common resource list method.":{{site.baseurl}}/api/methods.html#index -h2. show +h3. show show groups @@ -83,7 +94,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string||path|| -h2. update +h3. update Update attributes of an existing Group. diff --git a/doc/api/methods/humans.html.textile.liquid b/doc/api/methods/humans.html.textile.liquid index 1d8c13e9dc..c4f2fd6215 100644 --- a/doc/api/methods/humans.html.textile.liquid +++ b/doc/api/methods/humans.html.textile.liquid @@ -6,14 +6,29 @@ title: "humans" ... -See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html - API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/humans@ -Required arguments are displayed in %{background:#ccffcc}green%. +Object type: @7a9it@ + +Example UUID: @zzzzz-7a9it-0123456789abcde@ + +h2. Resource + +A metadata record that may be used to represent a human subject. + +Each Human has, in addition to the "Common resource fields":{{site.baseurl}}/api/resources.html: + +table(table table-bordered table-condensed). +|_. Attribute|_. Type|_. Description|_. Example| +|properties|hash||| +h2. Methods -h2. create +See "Common resource methods":{{site.baseurl}}/api/methods.html for more information about @create@, @delete@, @get@, @list@, and @update@. + +Required arguments are displayed in %{background:#ccffcc}green%. + +h3. create Create a new Human. @@ -23,7 +38,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | |human|object||query|| -h2. delete +h3. delete Delete an existing Human. @@ -33,7 +48,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the Human in question.|path|| -h2. get +h3. get Gets a Human's metadata by UUID. @@ -43,19 +58,13 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the Human in question.|path|| -h2. list +h3. list List humans. -Arguments: - -table(table table-bordered table-condensed). -|_. Argument |_. Type |_. Description |_. Location |_. Example | -|limit|integer (default 100)|Maximum number of humans to return.|query|| -|order|string|Order in which to return matching humans.|query|| -|filters|array|Conditions for filtering humans.|query|| +See "common resource list method.":{{site.baseurl}}/api/methods.html#index -h2. update +h3. update Update attributes of an existing Human. diff --git a/doc/api/methods/job_tasks.html.textile.liquid b/doc/api/methods/job_tasks.html.textile.liquid index 7b040d85b3..4969abf9ef 100644 --- a/doc/api/methods/job_tasks.html.textile.liquid +++ b/doc/api/methods/job_tasks.html.textile.liquid @@ -6,14 +6,45 @@ title: "job_tasks" ... -See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html - API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/job_tasks@ -Required arguments are displayed in %{background:#ccffcc}green%. +Object type: @ot0gb@ + +Example UUID: @zzzzz-ot0gb-0123456789abcde@ + +h2. Resource + +Deprecated. +A job task is a individually scheduled unit of work executed as part of an overall job. -h2. create +Each JobTask has, in addition to the "Common resource fields":{{site.baseurl}}/api/resources.html: + +table(table table-bordered table-condensed). +|_. Attribute|_. Type|_. Description|_. Example| +|sequence|integer|Execution sequence. +A step cannot be run until all steps with lower sequence numbers have completed. +Job steps with the same sequence number can be run in any order.|| +|parameters|hash||| +|output|text||| +|progress|float||| +|success|boolean|Is null if the task has neither completed successfully nor failed permanently.|| + +The following attributes should not be updated by anyone other than the job manager: + +table(table table-bordered table-condensed). +|_. Attribute|_. Type|_. Description|_. Notes| +|qsequence|integer|Order of arrival|0-based| +|job_uuid|string||| +|created_by_job_task_uuid|string||| + +h2. Methods + +See "Common resource methods":{{site.baseurl}}/api/methods.html for more information about @create@, @delete@, @get@, @list@, and @update@. + +Required arguments are displayed in %{background:#ccffcc}green%. + +h3. create Create a new JobTask. @@ -23,7 +54,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | |job_task|object||query|| -h2. delete +h3. delete Delete an existing JobTask. @@ -33,7 +64,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the JobTask in question.|path|| -h2. get +h3. get Gets a JobTask's metadata by UUID. @@ -43,19 +74,13 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the JobTask in question.|path|| -h2. list +h3. list List job_tasks. -Arguments: - -table(table table-bordered table-condensed). -|_. Argument |_. Type |_. Description |_. Location |_. Example | -|limit|integer (default 100)|Maximum number of job_tasks to return.|query|| -|order|string|Order in which to return matching job_tasks.|query|| -|filters|array|Conditions for filtering job_tasks.|query|| +See "common resource list method.":{{site.baseurl}}/api/methods.html#index -h2. update +h3. update Update attributes of an existing JobTask. diff --git a/doc/api/methods/jobs.html.textile.liquid b/doc/api/methods/jobs.html.textile.liquid index 90a3c4c722..01776ab178 100644 --- a/doc/api/methods/jobs.html.textile.liquid +++ b/doc/api/methods/jobs.html.textile.liquid @@ -6,13 +6,76 @@ title: "jobs" ... -See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html - API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/jobs@ +Object type: @8i9sb@ + +Example UUID: @zzzzz-8i9sb-0123456789abcde@ + +h2. Resource + +Deprecated. + +A job describes a work order to be executed by the Arvados cluster. + +Each job has, in addition to the "Common resource fields":{{site.baseurl}}/api/resources.html: + +table(table table-bordered table-condensed). +|_. Attribute|_. Type|_. Description|_. Notes| +|script|string|The filename of the job script.|This program will be invoked by Crunch for each job task. It is given as a path to an executable file, relative to the @/crunch_scripts@ directory in the Git tree specified by the _repository_ and _script_version_ attributes.| +|script_parameters|hash|The input parameters for the job.|Conventionally, one of the parameters is called @"input"@. Typically, some parameter values are collection UUIDs. Ultimately, though, the significance of parameters is left entirely up to the script itself.| +|repository|string|Git repository name or URL.|Source of the repository where the given script_version is to be found. This can be given as the name of a locally hosted repository, or as a publicly accessible URL starting with @git://@, @http://@, or @https://@. +Examples: +@yourusername/yourrepo@ +@https://github.com/curoverse/arvados.git@| +|script_version|string|Git commit|During a **create** transaction, this is the Git branch, tag, or hash supplied by the client. Before the job starts, Arvados updates it to the full 40-character SHA-1 hash of the commit used by the job. +See "Specifying Git versions":#script_version below for more detail about acceptable ways to specify a commit.| +|cancelled_by_client_uuid|string|API client ID|Is null if job has not been cancelled| +|cancelled_by_user_uuid|string|Authenticated user ID|Is null if job has not been cancelled| +|cancelled_at|datetime|When job was cancelled|Is null if job has not been cancelled| +|started_at|datetime|When job started running|Is null if job has not [yet] started| +|finished_at|datetime|When job finished running|Is null if job has not [yet] finished| +|running|boolean|Whether the job is running|| +|success|boolean|Whether the job indicated successful completion|Is null if job has not finished| +|is_locked_by_uuid|string|UUID of the user who has locked this job|Is null if job is not locked. The system user locks the job when starting the job, in order to prevent job attributes from being altered.| +|node_uuids|array|List of UUID strings for node objects that have been assigned to this job|| +|log|string|Collection UUID|Is null if the job has not finished. After the job runs, the given collection contains a text file with log messages provided by the @arv-crunch-job@ task scheduler as well as the standard error streams provided by the task processes.| +|tasks_summary|hash|Summary of task completion states.|Example: @{"done":0,"running":4,"todo":2,"failed":0}@| +|output|string|Collection UUID|Is null if the job has not finished.| +|nondeterministic|boolean|The job is expected to produce different results if run more than once.|If true, this job will not be considered as a candidate for automatic re-use when submitting subsequent identical jobs.| +|submit_id|string|Unique ID provided by client when job was submitted|Optional. This can be used by a client to make the "jobs.create":{{site.baseurl}}/api/methods/jobs.html#create method idempotent.| +|priority|string||| +|arvados_sdk_version|string|Git commit hash that specifies the SDK version to use from the Arvados repository|This is set by searching the Arvados repository for a match for the arvados_sdk_version runtime constraint.| +|docker_image_locator|string|Portable data hash of the collection that contains the Docker image to use|This is set by searching readable collections for a match for the docker_image runtime constraint.| +|runtime_constraints|hash|Constraints that must be satisfied by the job/task scheduler in order to run the job.|See below.| +|components|hash|Name and uuid pairs representing the child work units of this job. The uuids can be of different object types.|Example components hash: @{"name1": "zzzzz-8i9sb-xyz...", "name2": "zzzzz-d1hrv-xyz...",}@| + +h3(#script_version). Specifying Git versions + +The script_version attribute and arvados_sdk_version runtime constraint are typically given as a branch, tag, or commit hash, but there are many more ways to specify a Git commit. The "specifying revisions" section of the "gitrevisions manual page":http://git-scm.com/docs/gitrevisions.html has a definitive list. Arvados accepts Git versions in any format listed there that names a single commit (not a tree, a blob, or a range of commits). However, some kinds of names can be expected to resolve differently in Arvados than they do in your local repository. For example, HEAD@{1} refers to the local reflog, and @origin/master@ typically refers to a remote branch: neither is likely to work as desired if given as a Git version. + +h3. Runtime constraints + +table(table table-bordered table-condensed). +|_. Key|_. Type|_. Description|_. Implemented| +|arvados_sdk_version|string|The Git version of the SDKs to use from the Arvados git repository. See "Specifying Git versions":#script_version for more detail about acceptable ways to specify a commit. If you use this, you must also specify a @docker_image@ constraint (see below). In order to install the Python SDK successfully, Crunch must be able to find and run virtualenv inside the container.|✓| +|docker_image|string|The Docker image that this Job needs to run. If specified, Crunch will create a Docker container from this image, and run the Job's script inside that. The Keep mount and work directories will be available as volumes inside this container. The image must be uploaded to Arvados using @arv keep docker@. You may specify the image in any format that Docker accepts, such as @arvados/jobs@, @debian:latest@, or the Docker image id. Alternatively, you may specify the portable data hash of the image Collection.|✓| +|min_nodes|integer||✓| +|max_nodes|integer||| +|min_cores_per_node|integer|Require that each node assigned to this Job have the specified number of CPU cores|✓| +|min_ram_mb_per_node|integer|Require that each node assigned to this Job have the specified amount of real memory (in MiB)|✓| +|min_scratch_mb_per_node|integer|Require that each node assigned to this Job have the specified amount of scratch storage available (in MiB)|✓| +|max_tasks_per_node|integer|Maximum simultaneous tasks on a single node|✓| +|keep_cache_mb_per_task|integer|Size of file data buffer for per-task Keep directory ($TASK_KEEPMOUNT), in MiB. Default is 256 MiB. Increase this to reduce cache thrashing in situtations such as accessing multiple large (64+ MiB) files at the same time, or accessing different parts of a large file at the same time.|✓| +|min_ram_per_task|integer|Minimum real memory (KiB) per task|| + +h2. Methods + +See "Common resource methods":{{site.baseurl}}/api/methods.html for more information about @create@, @delete@, @get@, @list@, and @update@. + Required arguments are displayed in %{background:#ccffcc}green%. -h2. cancel +h3. cancel Cancel a job that is queued or running. @@ -22,7 +85,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string||path|| -h2(#create). create +h3(#create). create Create a new Job. @@ -30,7 +93,7 @@ Arguments: table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | -{background:#ccffcc}.|job|object|See "Job resource":{{site.baseurl}}/api/schema/Job.html|request body|| +{background:#ccffcc}.|job|object|Job resource|request body|| |minimum_script_version |string |Git branch, tag, or commit hash specifying the minimum acceptable script version (earliest ancestor) to consider when deciding whether to re-use a past job.[1]|query|@"c3e86c9"@| |exclude_script_versions|array of strings|Git commit branches, tags, or hashes to exclude when deciding whether to re-use a past job.|query|@["8f03c71","8f03c71"]@ @["badtag1","badtag2"]@| @@ -39,9 +102,9 @@ table(table table-bordered table-condensed). When a job is submitted to the queue using the **create** method, the @script_version@ attribute is updated to a full 40-character Git commit hash based on the current content of the specified repository. If @script_version@ cannot be resolved, the job submission is rejected. -fn1. See the "note about specifying Git commits on the Job resource page":{{site.baseurl}}/api/schema/Job.html#script_version for more detail. +fn1. See the "note about specifying Git commits":#script_version for more detail. -h3. Specialized filters +h4. Specialized filters Special filter operations are available for specific Job columns. @@ -53,7 +116,7 @@ Special filter operations are available for specific Job columns. * @docker_image_locator@ @not in docker@ @SEARCH@
Negate the @in docker@ filter. -h3. Reusing jobs +h4. Reusing jobs Because Arvados records the exact version of the script, input parameters, and runtime environment that was used to run the job, if the script is deterministic (meaning that the same code version is guaranteed to produce the same outputs from the same inputs) then it is possible to re-use the results of past jobs, and avoid re-running the computation to save time. Arvados uses the following algorithm to determine if a past job can be re-used: @@ -68,7 +131,7 @@ notextile.
-h3. Examples +h4. Examples Run the script "crunch_scripts/hash.py" in the repository "you" using the "master" commit. Arvados should re-use a previous job if the script_version of the previous job is the same as the current "master" commit. This works irrespective of whether the previous job was submitted using the name "master", a different branch name or tag indicating the same commit, a SHA-1 commit hash, etc. @@ -156,7 +219,7 @@ Run the script "crunch_scripts/monte-carlo.py" in the repository "you/you" using } -h2. delete +h3. delete Delete an existing Job. @@ -166,7 +229,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the Job in question.|path|| -h2. get +h3. get Gets a Job's metadata by UUID. @@ -176,21 +239,15 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the Job in question.|path|| -h2. list +h3. list List jobs. -Arguments: - -table(table table-bordered table-condensed). -|_. Argument |_. Type |_. Description |_. Location |_. Example | -|limit|integer (default 100)|Maximum number of jobs to return.|query|| -|order|string|Order in which to return matching jobs.|query|| -|filters|array|Conditions for filtering jobs.|query|| +See "common resource list method.":{{site.baseurl}}/api/methods.html#index See the create method documentation for more information about Job-specific filters. -h2. log_tail_follow +h3. log_tail_follow log_tail_follow jobs @@ -201,7 +258,7 @@ table(table table-bordered table-condensed). {background:#ccffcc}.|uuid|string||path|| |buffer_size|integer (default 8192)||query|| -h2. queue +h3. queue Get the current job queue. @@ -214,7 +271,7 @@ table(table table-bordered table-condensed). This method is equivalent to the "list method":#list, except that the results are restricted to queued jobs (i.e., jobs that have not yet been started or cancelled) and order defaults to queue priority. -h2. update +h3. update Update attributes of an existing Job. diff --git a/doc/api/methods/keep_disks.html.textile.liquid b/doc/api/methods/keep_disks.html.textile.liquid index 5179ccc8b2..421e942faa 100644 --- a/doc/api/methods/keep_disks.html.textile.liquid +++ b/doc/api/methods/keep_disks.html.textile.liquid @@ -2,17 +2,43 @@ layout: default navsection: api navmenu: API Methods -title: "keep_disks" +title: "keep_disks (deprecated)" ... -See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html - API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/keep_disks@ +Object type: @penuu@ + +Example UUID: @zzzzz-penuu-0123456789abcde@ + +h2. Resource + +Obsoleted by "keep_services":{{site.baseurl}}/api/methods/keep_services.html + +Each KeepDisk has, in addition to the "Common resource fields":{{site.baseurl}}/api/resources.html: + +table(table table-bordered table-condensed). +|_. Attribute|_. Type|_. Description|_. Example| +|ping_secret|string||| +|node_uuid|string||| +|filesystem_uuid|string||| +|bytes_total|integer||| +|bytes_free|integer||| +|is_readable|boolean||| +|is_writable|boolean||| +|last_read_at|datetime||| +|last_write_at|datetime||| +|last_ping_at|datetime||| +|keep_service_uuid|string||| + +h2. Methods + +See "Common resource methods":{{site.baseurl}}/api/methods.html for more information about @create@, @delete@, @get@, @list@, and @update@. + Required arguments are displayed in %{background:#ccffcc}green%. -h2. create +h3. create Create a new KeepDisk. @@ -22,7 +48,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | |keep_disk|object||query|| -h2. delete +h3. delete Delete an existing KeepDisk. @@ -32,7 +58,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the KeepDisk in question.|path|| -h2. get +h3. get Gets a KeepDisk's metadata by UUID. @@ -42,19 +68,13 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the KeepDisk in question.|path|| -h2. list +h3. list List keep_disks. -Arguments: - -table(table table-bordered table-condensed). -|_. Argument |_. Type |_. Description |_. Location |_. Example | -|limit|integer (default 100)|Maximum number of keep_disks to return.|query|| -|order|string|Order in which to return matching keep_disks.|query|| -|filters|array|Conditions for filtering keep_disks.|query|| +See "common resource list method.":{{site.baseurl}}/api/methods.html#index -h2. ping +h3. ping ping keep_disks @@ -70,7 +90,7 @@ table(table table-bordered table-condensed). |service_host|string||query|| |uuid|string||query|| -h2. update +h3. update Update attributes of an existing KeepDisk. diff --git a/doc/api/methods/keep_services.html.textile.liquid b/doc/api/methods/keep_services.html.textile.liquid index da6818b92d..dadf783324 100644 --- a/doc/api/methods/keep_services.html.textile.liquid +++ b/doc/api/methods/keep_services.html.textile.liquid @@ -6,22 +6,36 @@ title: "keep_services" ... -See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html - API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/keep_services@ -Required arguments are displayed in %{background:#ccffcc}green%. +Object type: @bi6l4@ + +Example UUID: @zzzzz-bi6l4-0123456789abcde@ + +h2. Resource + +The keep_services resource keep clients to discover storage servers and proxies available on the cluster for persistent storage and retrieval of keep blocks. + +Each KeepService has, in addition to the "Common resource fields":{{site.baseurl}}/api/resources.html: + +table(table table-bordered table-condensed). +|_. Attribute|_. Type|_. Description|_. Example| +|service_host|string|hostname of the server|| +|service_port|integer|TCP port of the service|| +|service_ssl_flag|boolean|if the server uses SSL|| +|service_type|string|The service type, one of "disk", "blob" (cloud object store) or "proxy" (keepproxy)|| -h2. accessible +h2. Methods -Get a list of keep services that are accessible to the requesting client. This -is context-sensitive, for example providing the list of actual Keep servers -when inside the cluster, but providing a proxy service if client contacts -Arvados from outside the cluster. +See "Common resource methods":{{site.baseurl}}/api/methods.html for more information about @create@, @delete@, @get@, @list@, and @update@. -Takes no arguments. +Required arguments are displayed in %{background:#ccffcc}green%. + +h3. accessible -h2. create +Get a list of keep services that are accessible to the requesting client. Unlike @list@, this is context-sensitive based on the requester, for example providing the list of actual Keep servers when inside the cluster, but providing a proxy service if client contacts Arvados from outside the cluster. + +h3. create Create a new KeepService. @@ -29,9 +43,9 @@ Arguments: table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | -|keep_disk|object||query|| +|keep_service|object||query|| -h2. delete +h3. delete Delete an existing KeepService. @@ -41,7 +55,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the KeepService in question.|path|| -h2. get +h3. get Gets a KeepService's metadata by UUID. @@ -51,19 +65,13 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the KeepService in question.|path|| -h2. list +h3. list List keep_services. -Arguments: - -table(table table-bordered table-condensed). -|_. Argument |_. Type |_. Description |_. Location |_. Example | -|limit|integer (default 100)|Maximum number of keep_services to return.|query|| -|order|string|Order in which to return matching keep_services.|query|| -|filters|array|Conditions for filtering keep_services.|query|| +See "common resource list method.":{{site.baseurl}}/api/methods.html#index -h2. update +h3. update Update attributes of an existing KeepService. diff --git a/doc/api/methods/links.html.textile.liquid b/doc/api/methods/links.html.textile.liquid index 3c0bdf346d..d94a055150 100644 --- a/doc/api/methods/links.html.textile.liquid +++ b/doc/api/methods/links.html.textile.liquid @@ -6,13 +6,50 @@ title: "links" ... -See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html - API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/links@ +Object type: @o0j2j@ + +Example UUID: @zzzzz-o0j2j-0123456789abcde@ + +h2. Resource + +Links are an extensible way to describe relationships between Arvados objects and metadata about individual objects. + +Each link has, in addition to the "Common resource fields":{{site.baseurl}}/api/resources.html: + +table(table table-bordered table-condensed). +|_. Attribute|_. Type|_. Description| +|head_uuid|string|The object being described or acted on.| +|tail_uuid|string|The origin or actor in the description or action (may be null).| +|link_class|string|Type of link| +|name|string|Primary value of the link.| +|properties|hash|Additional information, expressed as a key→value hash. Key: string. Value: string, number, array, or hash.| + +h2. Link classes + +Some classes are pre-defined by convention and have standard meanings attached to names. + +h3. permission + +See "permission links":{{site.baseurl}}/api/permission-model.html#links section of the permission model. + +h3. tag + +A **tag** link describes an object using an unparsed plain text string. Tags can be used to annotate objects that are not editable, like collections and objects shared as read-only. + +table(table table-bordered table-condensed). +|_. tail_type→head_type|_. name→head_uuid {properties}| +|→Collection | _tag name_ → _collection uuid_| +|→Job | _tag name_ → _job uuid_| + +h2. Methods + +See "Common resource methods":{{site.baseurl}}/api/methods.html for more information about @create@, @delete@, @get@, @list@, and @update@. + Required arguments are displayed in %{background:#ccffcc}green%. -h2. create +h3. create Create a new Link. @@ -22,7 +59,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | |link|object||query|| -h2. delete +h3. delete Delete an existing Link. @@ -32,7 +69,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the Link in question.|path|| -h2. get +h3. get Gets a Link's metadata by UUID. @@ -42,29 +79,13 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the Link in question.|path|| -h2. list +h3. list List links. -Arguments: - -table(table table-bordered table-condensed). -|_. Argument |_. Type |_. Description |_. Location |_. Example | -|limit|integer (default 100)|Maximum number of links to return.|query|| -|order|string|Order in which to return matching links.|query|| -|filters|array|Conditions for filtering links.|query|| - -h2. render_not_found - -render_not_found links - -Arguments: - -table(table table-bordered table-condensed). -|_. Argument |_. Type |_. Description |_. Location |_. Example | -{background:#ccffcc}.|a|string||path|| +See "common resource list method.":{{site.baseurl}}/api/methods.html#index -h2. update +h3. update Update attributes of an existing Link. diff --git a/doc/api/methods/logs.html.textile.liquid b/doc/api/methods/logs.html.textile.liquid index c5895d78a2..e5380d8939 100644 --- a/doc/api/methods/logs.html.textile.liquid +++ b/doc/api/methods/logs.html.textile.liquid @@ -6,14 +6,39 @@ title: "logs" ... -See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html - API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/logs@ -Required arguments are displayed in %{background:#ccffcc}green%. +Object type: @57u5n@ + +Example UUID: @zzzzz-57u5n-0123456789abcde@ + +h2. Resource + +Each Log has, in addition to the "Common resource fields":{{site.baseurl}}/api/resources.html: + +table(table table-bordered table-condensed). +|_. Attribute|_. Type|_. Description|_. Example| +|object_uuid|string|The arvados object that is the subject of the log.|| +|event_at|datetime||| +|event_type|string|A user-defined category or type for this event.|@LOGIN@| +|summary|text||| +|properties|hash||| + +h3. Creation + +Any user may create Log entries for any event they find useful. User-generated Logs have no intrinsic meaning to other users or to the Arvados system itself; it is up to each user to choose appropriate log event types and summaries for their project. +h3. System Logs -h2. create +Arvados uses Logs to record creation, deletion, and updates of other Arvados resources. + +h2. Methods + +See "Common resource methods":{{site.baseurl}}/api/methods.html for more information about @create@, @delete@, @get@, @list@, and @update@. + +Required arguments are displayed in %{background:#ccffcc}green%. + +h3. create Create a new log entry. @@ -23,7 +48,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | |log|object||query|| -h2. delete +h3. delete Delete an existing log entry. This method can only be used by privileged (system administrator) users. @@ -33,7 +58,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the log entry in question.|path|| -h2. get +h3. get Retrieve a log entry. @@ -43,19 +68,13 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the log entry in question.|path|| -h2. list +h3. list List log entries. -Arguments: - -table(table table-bordered table-condensed). -|_. Argument |_. Type |_. Description |_. Location |_. Example | -|limit|integer (default 100)|Maximum number of log entries to return.|query|| -|order|string|Order in which to return matching log entries.|query|| -|filters|array|Conditions for filtering log entries.|query|| +See "common resource list method.":{{site.baseurl}}/api/methods.html#index -h2. update +h3. update Update attributes of an existing log entry. This method can only be used by privileged (system administrator) users. diff --git a/doc/api/methods/nodes.html.textile.liquid b/doc/api/methods/nodes.html.textile.liquid index 7aa5896ea5..aa21237bfa 100644 --- a/doc/api/methods/nodes.html.textile.liquid +++ b/doc/api/methods/nodes.html.textile.liquid @@ -6,14 +6,36 @@ title: "nodes" ... -See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html - API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/nodes@ -Required arguments are displayed in %{background:#ccffcc}green%. +Object type: @7ekkf@ + +Example UUID: @zzzzz-7ekkf-0123456789abcde@ + +h2. Resource + +Node resources list compute nodes on which Crunch may schedule work. + +Each Node has, in addition to the "Common resource fields":{{site.baseurl}}/api/resources.html: + +table(table table-bordered table-condensed). +|_. Attribute|_. Type|_. Description|_. Example| +|slot_number|integer||| +|hostname|string||| +|domain|string||| +|ip_address|string||| +|job_uuid|string|The UUID of the job that this node is assigned to work on. If you do not have permission to read the job, this will be null.|| +|first_ping_at|datetime||| +|last_ping_at|datetime||| +|info|hash||| +h2. Methods -h2. create +See "Common resource methods":{{site.baseurl}}/api/methods.html for more information about @create@, @delete@, @get@, @list@, and @update@. + +Required arguments are displayed in %{background:#ccffcc}green%. + +h3. create Create a new Node. @@ -23,7 +45,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|node|object||query|| -h2. delete +h3. delete Delete an existing Node. @@ -33,7 +55,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the Node in question.|path|| -h2. get +h3. get Gets a Node's metadata by UUID. @@ -43,21 +65,15 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the Node in question.|path|| -h2. list +h3. list List nodes. -Arguments: - -table(table table-bordered table-condensed). -|_. Argument |_. Type |_. Description |_. Location |_. Example | -|limit|integer (default 100)|Maximum number of nodes to return.|query|| -|order|string|Order in which to return matching nodes.|query|| -|filters|array|Conditions for filtering nodes.|query|| +See "common resource list method.":{{site.baseurl}}/api/methods.html#index -h2. ping +h3. ping -ping nodes +Process a ping from a compute node. Arguments: @@ -66,7 +82,7 @@ table(table table-bordered table-condensed). {background:#ccffcc}.|ping_secret|string||query|| {background:#ccffcc}.|uuid|string||path|| -h2. update +h3. update Update attributes of an existing Node. diff --git a/doc/api/methods/pipeline_instances.html.textile.liquid b/doc/api/methods/pipeline_instances.html.textile.liquid index d637a696eb..f4fb774cf4 100644 --- a/doc/api/methods/pipeline_instances.html.textile.liquid +++ b/doc/api/methods/pipeline_instances.html.textile.liquid @@ -6,14 +6,34 @@ title: "pipeline_instances" ... -See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html - API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/pipeline_instances@ -Required arguments are displayed in %{background:#ccffcc}green%. +Object type: @d1hrv@ + +Example UUID: @zzzzz-d1hrv-0123456789abcde@ + +h2. Resource + +Deprecated. A pipeline instance is a collection of jobs managed by @aravdos-run-pipeline-instance@. + +Each PipelineInstance has, in addition to the "Common resource fields":{{site.baseurl}}/api/resources.html: + +table(table table-bordered table-condensed). +|_. Attribute|_. Type|_. Description|_. Example| +|pipeline_template_uuid|string|The "pipeline template":pipeline_templates.html that this instance was created from.|| +|name|string||| +|components|hash||| +|success|boolean||| +|active|boolean||| +|properties|Hash||| +h2. Methods -h2. create +See "Common resource methods":{{site.baseurl}}/api/methods.html for more information about @create@, @delete@, @get@, @list@, and @update@. + +Required arguments are displayed in %{background:#ccffcc}green%. + +h3. create Create a new PipelineInstance. @@ -23,7 +43,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | |pipeline_instance|object||query|| -h2. delete +h3. delete Delete an existing PipelineInstance. @@ -33,7 +53,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the PipelineInstance in question.|path|| -h2. get +h3. get Gets a PipelineInstance's metadata by UUID. @@ -43,19 +63,13 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the PipelineInstance in question.|path|| -h2. list +h3. list List pipeline_instances. -Arguments: - -table(table table-bordered table-condensed). -|_. Argument |_. Type |_. Description |_. Location |_. Example | -|limit|integer (default 100)|Maximum number of pipeline_instances to return.|query|| -|order|string|Order in which to return matching pipeline_instances.|query|| -|filters|array|Conditions for filtering pipeline_instances.|query|| +See "common resource list method.":{{site.baseurl}}/api/methods.html#index -h2. update +h3. update Update attributes of an existing PipelineInstance. diff --git a/doc/api/methods/pipeline_templates.html.textile.liquid b/doc/api/methods/pipeline_templates.html.textile.liquid index 06684cc6df..e8fb6a4c59 100644 --- a/doc/api/methods/pipeline_templates.html.textile.liquid +++ b/doc/api/methods/pipeline_templates.html.textile.liquid @@ -6,14 +6,172 @@ title: "pipeline_templates" ... -See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html - API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/pipeline_templates@ -Required arguments are displayed in %{background:#ccffcc}green%. +Object type: @p5p6p@ + +Example UUID: @zzzzz-p5p6p-0123456789abcde@ + +h2. Resource + +Deprecated. A pipeline template is a collection of jobs that can be instantiated as a pipeline_instance. + +Each PipelineTemplate has, in addition to the "Common resource fields":{{site.baseurl}}/api/resources.html: + +table(table table-bordered table-condensed). +|_. Attribute|_. Type|_. Description|_. Example| +|name|string||| +|components|hash||| + +The pipeline template consists of "name" and "components". + +table(table table-bordered table-condensed). +|_. Attribute |_. Type |_. Accepted values |_. Required|_. Description| +|name |string |any |yes |The human-readable name of the pipeline template.| +|components |object |JSON object containing job submission objects|yes |The component jobs that make up the pipeline, with the component name as the key. | + +h3. Components +The components field of the pipeline template is a JSON object which describes the individual steps that make up the pipeline. Each component is an Arvados job submission. "Parameters for job submissions are described on the job method page.":{{site.baseurl}}/api/methods/jobs.html#create In addition, a component can have the following parameters: -h2. create +table(table table-bordered table-condensed). +|_. Attribute |_. Type |_. Accepted values |_. Required|_. Description| +|output_name |string or boolean|string or false |no |If a string is provided, use this name for the output collection of this component. If the value is false, do not create a permanent output collection (an temporary intermediate collection will still be created). If not provided, a default name will be assigned to the output.| + +h3. Script parameters + +When used in a pipeline, each parameter in the 'script_parameters' attribute of a component job can specify that the input parameter must be supplied by the user, or the input parameter should be linked to the output of another component. To do this, the value of the parameter should be JSON object containing one of the following attributes: + +table(table table-bordered table-condensed). +|_. Attribute |_. Type |_. Accepted values |_. Description| +|default |any |any |The default value for this parameter.| +|required |boolean |true or false |Specifies whether the parameter is required to have a value or not.| +|dataclass |string |One of 'Collection', 'File' [1], 'number', or 'text' |Data type of this parameter.| +|search_for |string |any string |Substring to use as a default search string when choosing inputs.| +|output_of |string |the name of another component in the pipeline |Specifies that the value of this parameter should be set to the 'output' attribute of the job that corresponds to the specified component.| +|title |string |any string |User friendly title to display when choosing parameter values| +|description |string |any string |Extended text description for describing expected/valid values for the script parameter| +|link_name |string |any string |User friendly name to display for the parameter value instead of the actual parameter value| + +The 'output_of' parameter is especially important, as this is how components are actually linked together to form a pipeline. Component jobs that depend on the output of other components do not run until the parent job completes and has produced output. If the parent job fails, the entire pipeline fails. + +fn1. The 'File' type refers to a specific file within a Keep collection in the form 'collection_hash/filename', for example '887cd41e9c613463eab2f0d885c6dd96+83/bob.txt'. + +The 'search_for' parameter is meaningful only when input dataclass of type Collection or File is used. If a value is provided, this will be preloaded into the input data chooser dialog in Workbench. For example, if your input dataclass is a File and you are interested in a certain filename extention, you can preconfigure it in this attribute. + +h3. Examples + +This is a pipeline named "Filter MD5 hash values" with two components, "do_hash" and "filter". The "input" script parameter of the "do_hash" component is required to be filled in by the user, and the expected data type is "Collection". This also specifies that the "input" script parameter of the "filter" component is the output of "do_hash", so "filter" will not run until "do_hash" completes successfully. When the pipeline runs, past jobs that meet the criteria described above may be substituted for either or both components to avoid redundant computation. + +
+{
+  "name": "Filter MD5 hash values",
+  "components": {
+    "do_hash": {
+      "script": "hash.py",
+      "repository": "you/you",
+      "script_version": "master",
+      "script_parameters": {
+        "input": {
+          "required": true,
+          "dataclass": "Collection",
+          "search_for": ".fastq.gz",
+          "title":"Please select a fastq file"
+        }
+      },
+    },
+    "filter": {
+      "script": "0-filter.py",
+      "repository": "you/you",
+      "script_version": "master",
+      "script_parameters": {
+        "input": {
+          "output_of": "do_hash"
+        }
+      },
+    }
+  }
+}
+
 + +This pipeline consists of three components. The components "thing1" and "thing2" both depend on "cat_in_the_hat". Once the "cat_in_the_hat" job is complete, both "thing1" and "thing2" can run in parallel, because they do not depend on each other. + +
+{
+  "name": "Wreck the house",
+  "components": {
+    "cat_in_the_hat": {
+      "script": "cat.py",
+      "repository": "you/you",
+      "script_version": "master",
+      "script_parameters": { }
+    },
+    "thing1": {
+      "script": "thing1.py",
+      "repository": "you/you",
+      "script_version": "master",
+      "script_parameters": {
+        "input": {
+          "output_of": "cat_in_the_hat"
+        }
+      },
+    },
+    "thing2": {
+      "script": "thing2.py",
+      "repository": "you/you",
+      "script_version": "master",
+      "script_parameters": {
+        "input": {
+          "output_of": "cat_in_the_hat"
+        }
+      },
+    },
+  }
+}
+
 + +This pipeline consists of three components. The component "cleanup" depends on "thing1" and "thing2". Both "thing1" and "thing2" are started immediately and can run in parallel, because they do not depend on each other, but "cleanup" cannot begin until both "thing1" and "thing2" have completed. + +
+{
+  "name": "Clean the house",
+  "components": {
+    "thing1": {
+      "script": "thing1.py",
+      "repository": "you/you",
+      "script_version": "master",
+      "script_parameters": { }
+    },
+    "thing2": {
+      "script": "thing2.py",
+      "repository": "you/you",
+      "script_version": "master",
+      "script_parameters": { }
+    },
+    "cleanup": {
+      "script": "cleanup.py",
+      "repository": "you/you",
+      "script_version": "master",
+      "script_parameters": {
+        "mess1": {
+          "output_of": "thing1"
+        },
+        "mess2": {
+          "output_of": "thing2"
+        }
+      }
+    }
+  }
+}
+
 + +h2. Methods + +See "Common resource methods":{{site.baseurl}}/api/methods.html for more information about @create@, @delete@, @get@, @list@, and @update@. + +Required arguments are displayed in %{background:#ccffcc}green%. + +h3. create Create a new PipelineTemplate. @@ -23,7 +181,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | |pipeline_template|object||query|| -h2. delete +h3. delete Delete an existing PipelineTemplate. @@ -33,7 +191,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the PipelineTemplate in question.|path|| -h2. get +h3. get Gets a PipelineTemplate's metadata by UUID. @@ -43,19 +201,13 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the PipelineTemplate in question.|path|| -h2. list +h3. list List pipeline_templates. -Arguments: - -table(table table-bordered table-condensed). -|_. Argument |_. Type |_. Description |_. Location |_. Example | -|limit|integer (default 100)|Maximum number of pipeline_templates to return.|query|| -|order|string|Order in which to return matching pipeline_templates.|query|| -|filters|array|Conditions for filtering pipeline_templates.|query|| +See "common resource list method.":{{site.baseurl}}/api/methods.html#index -h2. update +h3. update Update attributes of an existing PipelineTemplate. diff --git a/doc/api/methods/repositories.html.textile.liquid b/doc/api/methods/repositories.html.textile.liquid index 7bd8dd9d42..80dfc0b49c 100644 --- a/doc/api/methods/repositories.html.textile.liquid +++ b/doc/api/methods/repositories.html.textile.liquid @@ -6,14 +6,33 @@ title: "repositories" ... -See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html - API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/repositories@ -Required arguments are displayed in %{background:#ccffcc}green%. +Object type: @s0uqq@ + +Example UUID: @zzzzz-s0uqq-0123456789abcde@ + +h2. Resource + +The repositories resource lists git repositories managed by Arvados. + +Each Repository has, in addition to the "Common resource fields":{{site.baseurl}}/api/resources.html: + +table(table table-bordered table-condensed). +|_. Attribute|_. Type|_. Description|_. Example| +|name|string|The name of the repository on disk. Repository names must begin with a letter and contain only alphanumerics. Unless the repository is owned by the system user, the name must begin with the owner's username, then be separated from the base repository name with @/@. You may not create a repository that is owned by a user without a username.|@username/project1@| +|clone_urls|array|URLs from which the repository can be cloned. Read-only.|@["git@git.zzzzz.arvadosapi.com:foo/bar.git", + "https://git.zzzzz.arvadosapi.com/foo/bar.git"]@| +|fetch_url|string|URL suggested as a fetch-url in git config. Deprecated. Read-only.|| +|push_url|string|URL suggested as a push-url in git config. Deprecated. Read-only.|| +h2. Methods -h2. create +See "Common resource methods":{{site.baseurl}}/api/methods.html for more information about @create@, @delete@, @get@, @list@, and @update@. + +Required arguments are displayed in %{background:#ccffcc}green%. + +h3. create Create a new Repository. @@ -23,7 +42,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | |repository|object||query|| -h2. delete +h3. delete Delete an existing Repository. @@ -33,7 +52,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the Repository in question.|path|| -h2. get +h3. get Gets a Repository's metadata by UUID. @@ -43,7 +62,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the Repository in question.|path|| -h2. get_all_permissions +h3. get_all_permissions get_all_permissions repositories @@ -52,19 +71,13 @@ Arguments: table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | -h2. list +h3. list List repositories. -Arguments: - -table(table table-bordered table-condensed). -|_. Argument |_. Type |_. Description |_. Location |_. Example | -|limit|integer (default 100)|Maximum number of repositories to return.|query|| -|order|string|Order in which to return matching repositories.|query|| -|filters|array|Conditions for filtering repositories.|query|| +See "common resource list method.":{{site.baseurl}}/api/methods.html#index -h2. update +h3. update Update attributes of an existing Repository. diff --git a/doc/api/methods/specimens.html.textile.liquid b/doc/api/methods/specimens.html.textile.liquid index 6737c9bcae..652497d3f9 100644 --- a/doc/api/methods/specimens.html.textile.liquid +++ b/doc/api/methods/specimens.html.textile.liquid @@ -3,17 +3,32 @@ layout: default navsection: api navmenu: API Methods title: "specimens" - ... -See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html - API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/specimens@ -Required arguments are displayed in %{background:#ccffcc}green%. +Object type: @j58dm@ + +Example UUID: @zzzzz-j58dm-0123456789abcde@ + +h2. Resource + +A metadata record that may be used to represent a biological specimen. + +Each Specimen has, in addition to the "Common resource fields":{{site.baseurl}}/api/resources.html: + +table(table table-bordered table-condensed). +|_. Attribute|_. Type|_. Description|_. Example| +|material|string||| +|properties|hash||| + +h2. Methods +See "Common resource methods":{{site.baseurl}}/api/methods.html for more information about @create@, @delete@, @get@, @list@, and @update@. + +Required arguments are displayed in %{background:#ccffcc}green%. -h2. create +h3. create Create a new Specimen. @@ -23,7 +38,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | |specimen|object||query|| -h2. delete +h3. delete Delete an existing Specimen. @@ -33,7 +48,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the Specimen in question.|path|| -h2. get +h3. get Gets a Specimen's metadata by UUID. @@ -43,19 +58,13 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the Specimen in question.|path|| -h2. list +h3. list List specimens. -Arguments: - -table(table table-bordered table-condensed). -|_. Argument |_. Type |_. Description |_. Location |_. Example | -|limit|integer (default 100)|Maximum number of specimens to return.|query|| -|order|string|Order in which to return matching specimens.|query|| -|filters|array|Conditions for filtering specimens.|query|| +See "common resource list method.":{{site.baseurl}}/api/methods.html#index -h2. update +h3. update Update attributes of an existing Specimen. diff --git a/doc/api/methods/traits.html.textile.liquid b/doc/api/methods/traits.html.textile.liquid index 9b19a0880b..3f7d49f691 100644 --- a/doc/api/methods/traits.html.textile.liquid +++ b/doc/api/methods/traits.html.textile.liquid @@ -6,14 +6,30 @@ title: "traits" ... -See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html - API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/traits@ -Required arguments are displayed in %{background:#ccffcc}green%. +Object type: @q1cn2@ + +Example UUID: @zzzzz-q1cn2-0123456789abcde@ + +h2. Resource + +A metadata record that may be used to represent a genotype or phenotype trait. + +Each Trait has, in addition to the "Common resource fields":{{site.baseurl}}/api/resources.html: + +table(table table-bordered table-condensed). +|_. Attribute|_. Type|_. Description|_. Example| +|name|string||| +|properties|hash||| +h2. Methods -h2. create +See "Common resource methods":{{site.baseurl}}/api/methods.html for more information about @create@, @delete@, @get@, @list@, and @update@. + +Required arguments are displayed in %{background:#ccffcc}green%. + +h3. create Create a new Trait. @@ -23,7 +39,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | |trait|object||query|| -h2. delete +h3. delete Delete an existing Trait. @@ -33,7 +49,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the Trait in question.|path|| -h2. get +h3. get Gets a Trait's metadata by UUID. @@ -43,19 +59,13 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the Trait in question.|path|| -h2. list +h3. list List traits. -Arguments: - -table(table table-bordered table-condensed). -|_. Argument |_. Type |_. Description |_. Location |_. Example | -|limit|integer (default 100)|Maximum number of traits to return.|query|| -|order|string|Order in which to return matching traits.|query|| -|filters|array|Conditions for filtering traits.|query|| +See "common resource list method.":{{site.baseurl}}/api/methods.html#index -h2. update +h3. update Update attributes of an existing Trait. diff --git a/doc/api/methods/users.html.textile.liquid b/doc/api/methods/users.html.textile.liquid index 33f884b6cc..451205b48e 100644 --- a/doc/api/methods/users.html.textile.liquid +++ b/doc/api/methods/users.html.textile.liquid @@ -3,17 +3,40 @@ layout: default navsection: api navmenu: API Methods title: "users" - ... -See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html - API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/users@ -Required arguments are displayed in %{background:#ccffcc}green%. +Object type: @tpzed@ +Example UUID: @zzzzz-tpzed-0123456789abcde@ -h2. create +h2. Resource + +Users represent individuals with access to the Arvados cluster. + +Each User has, in addition to the "Common resource fields":{{site.baseurl}}/api/resources.html: + +table(table table-bordered table-condensed). +|_. Attribute|_. Type|_. Description|_. Example| +|email|string||| +|username|string|The username used for the user's git repositories and virtual machine logins. Usernames must start with a letter, and contain only alphanumerics. When a new user is created, a default username is set from their e-mail address. Only administrators may change the username.|| +|first_name|string||| +|last_name|string||| +|identity_url|string||| +|is_admin|boolean||| +|prefs|hash||| +|default_owner_uuid|string||| +|is_active|boolean||| +|writable_by|array|List of UUID strings identifying Groups and other Users that can modify this User object. This will include the user's owner_uuid and, for administrators and users requesting their own User object, the requesting user's UUID.|| + +h2. Methods + +See "Common resource methods":{{site.baseurl}}/api/methods.html for more information about @create@, @delete@, @get@, @list@, and @update@. + +Required arguments are displayed in %{background:#ccffcc}green%. + +h3. create Create a new User. @@ -23,16 +46,16 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | |user|object||query|| -h2. current +h3. current -current users +Get the user associated with the provided API token. Arguments: table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | -h2. delete +h3. delete Delete an existing User. @@ -48,7 +71,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string||path|| -h2. get +h3. get Gets a User's metadata by UUID. @@ -58,38 +81,22 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the User in question.|path|| -h2. list +h3. list List users. -Arguments: - -table(table table-bordered table-condensed). -|_. Argument |_. Type |_. Description |_. Location |_. Example | -|limit|integer (default 100)|Maximum number of users to return.|query|| -|order|string|Order in which to return matching users.|query|| -|filters|array|Conditions for filtering users.|query|| - -h2. show - -show users - -Arguments: - -table(table table-bordered table-condensed). -|_. Argument |_. Type |_. Description |_. Location |_. Example | -{background:#ccffcc}.|uuid|string||path|| +See "common resource list method.":{{site.baseurl}}/api/methods.html#index -h2. system +h3. system -system users +Get the user record for the "system user.":{{site.baseurl}}/api/permission-model.html#system Arguments: table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | -h2. update +h3. update Update attributes of an existing User. diff --git a/doc/api/methods/virtual_machines.html.textile.liquid b/doc/api/methods/virtual_machines.html.textile.liquid index 390abdc595..cad9bcbb1a 100644 --- a/doc/api/methods/virtual_machines.html.textile.liquid +++ b/doc/api/methods/virtual_machines.html.textile.liquid @@ -3,17 +3,31 @@ layout: default navsection: api navmenu: API Methods title: "virtual_machines" - ... -See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html - API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/virtual_machines@ -Required arguments are displayed in %{background:#ccffcc}green%. +Object type: @2x53u@ + +Example UUID: @zzzzz-2x53u-0123456789abcde@ + +h2. Resource + +The virtual_machines resource lists compute resources in the Arvados cluster to which a user may log in to get an interactive shell (via ssh or webshell). + +Each VirtualMachine has, in addition to the "Common resource fields":{{site.baseurl}}/api/resources.html: + +table(table table-bordered table-condensed). +|_. Attribute|_. Type|_. Description|_. Example| +|hostname|string||| + +h2. Methods +See "Common resource methods":{{site.baseurl}}/api/methods.html for more information about @create@, @delete@, @get@, @list@, and @update@. + +Required arguments are displayed in %{background:#ccffcc}green%. -h2. create +h3. create Create a new VirtualMachine. @@ -23,7 +37,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | |virtual_machine|object||query|| -h2. delete +h3. delete Delete an existing VirtualMachine. @@ -33,7 +47,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the VirtualMachine in question.|path|| -h2. get +h3. get Gets a VirtualMachine's metadata by UUID. @@ -43,7 +57,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the VirtualMachine in question.|path|| -h2(#logins). logins +h3(#logins). logins Get a list of SSH keys and account names that should be able to log in to a given virtual machine. @@ -53,7 +67,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string||path|| -The response is a "resource list":{{site.baseurl}}/api/resources.html#resourceList with @kind@ set to @"arvados#HashList"@. Each item is a hash with the following keys: +The response is an object with the field @items@ containing an array of objects in the following format: table(table table-bordered table-condensed). |_. Key|_. Value type|_. Description|_. Example| @@ -61,30 +75,24 @@ table(table table-bordered table-condensed). |hostname|string|Hostname of the virtual machine|@"shell.xyzzy.arvadosapi.com"@| |public_key|string|SSH public key|@"ssh-rsa AAAAB3NzaC1yc2E..."@| |user_uuid|string|UUID of the user who should be able to log in|@"xyzzy-tpzed-mv4d7dy7n91te11"@| -|virtual_machine_uuid|string|UUID of the "VirtualMachine resource":{{site.baseurl}}/api/schema/VirtualMachine.html|@"xyzzy-2x53u-kvszmclnbjuv8xc"@| -|authorized_key_uuid|string|UUID of the "AuthorizedKey resource":{{site.baseurl}}/api/schema/AuthorizedKey.html|@"xyzzy-fngyi-v9p0cyfmjxbio64"@| +|virtual_machine_uuid|string|UUID of the "virtual machine resource":{{site.baseurl}}/api/methods/virtual_machines.html|@"zzzzz-2x53u-kvszmclnbjuv8xc"@| +|authorized_key_uuid|string|UUID of the "authorized key resource":{{site.baseurl}}/api/methods/authorized_keys.html|@"zzzzz-fngyi-v9p0cyfmjxbio64"@| -h2. get_all_logins +h3. get_all_logins -Get a list, for every virtual machine in the system, of SSH keys and account names that should be able to log in. +Get a list of SSH keys and account names that should be able to log in for every virtual machine in the system. Arguments: none. The response has the same format as the response to the "logins method":#logins above. -h2. list +h3. list List virtual_machines. -Arguments: - -table(table table-bordered table-condensed). -|_. Argument |_. Type |_. Description |_. Location |_. Example | -|limit|integer (default 100)|Maximum number of virtual_machines to return.|query|| -|order|string|Order in which to return matching virtual_machines.|query|| -|filters|array|Conditions for filtering virtual_machines.|query|| +See "common resource list method.":{{site.baseurl}}/api/methods.html#index -h2. update +h3. update Update attributes of an existing VirtualMachine. diff --git a/doc/api/methods/workflows.html.textile.liquid b/doc/api/methods/workflows.html.textile.liquid index 95be013b62..57b301a3b9 100644 --- a/doc/api/methods/workflows.html.textile.liquid +++ b/doc/api/methods/workflows.html.textile.liquid @@ -3,17 +3,33 @@ layout: default navsection: api navmenu: API Methods title: "workflows" - ... -See "REST methods for working with Arvados resources":{{site.baseurl}}/api/methods.html - API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/workflows@ -Required arguments are displayed in %{background:#ccffcc}green%. +Object type: @7fd4e@ + +Example UUID: @zzzzz-7fd4e-0123456789abcde@ + +h2. Resource + +Stores a "Common Workflow Language":http://commonwl.org (CWL) computational workflow that can be searched for, browsed and executed (submitted to Crunch) from the workbench. + +Each Workflow offers the following optional attributes, in addition to the "Common resource fields":{{site.baseurl}}/api/resources.html: + +table(table table-bordered table-condensed). +|_. Attribute|_. Type|_. Description|_. Example| +|name|string|If not specified, will be set to any "name" from the "definition" attribute.|| +|description|string|If not specified, will be set to any "description" from the "definition" attribute.|| +|definition|string|A "Common Workflow Language" document.|Visit "Common Workflow Language":http://www.commonwl.org/ for details.| + +h2. Methods +See "Common resource methods":{{site.baseurl}}/api/methods.html for more information about @create@, @delete@, @get@, @list@, and @update@. + +Required arguments are displayed in %{background:#ccffcc}green%. -h2. create +h3. create Create a new Workflow. @@ -21,9 +37,9 @@ Arguments: table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | -{background:#ccffcc}.|workflow|object|See "Workflow resource":{{site.baseurl}}/api/schema/Workflow.html|request body|| +{background:#ccffcc}.|workflow|object|Workflow resource|request body|| -h2. delete +h3. delete Delete an existing Workflow. @@ -33,7 +49,7 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the Workflow in question.|path|| -h2. get +h3. get Get a Workflow's metadata by UUID. @@ -43,19 +59,13 @@ table(table table-bordered table-condensed). |_. Argument |_. Type |_. Description |_. Location |_. Example | {background:#ccffcc}.|uuid|string|The UUID of the Workflow in question.|path|| -h2. list +h3. list List workflows. -Arguments: - -table(table table-bordered table-condensed). -|_. Argument |_. Type |_. Description |_. Location |_. Example | -|limit|integer (default 100)|Maximum number of workflows to return.|query|| -|order|string|Order in which to return matching workflows.|query|| -|filters|array|Conditions for filtering workflows.|query|| +See "common resource list method.":{{site.baseurl}}/api/methods.html#index -h2. update +h3. update Update attributes of an existing Workflow. diff --git a/doc/api/permission-model.html.textile.liquid b/doc/api/permission-model.html.textile.liquid index 8b085ee5aa..62cd74d9d7 100644 --- a/doc/api/permission-model.html.textile.liquid +++ b/doc/api/permission-model.html.textile.liquid @@ -3,123 +3,70 @@ layout: default navsection: api navmenu: Concepts title: "Permission model" - ... +* There are four levels of permission: *none*, *can_read*, *can_write*, and *can_manage*. +** *none* is the default state when there are no other permission grants. +*** the object is not included in any list query response. +*** direct queries of the object by uuid return 404 Not Found. +*** Link objects require valid identifiers in @head_uuid@ and @tail_uuid@, so an attempt to create a Link that references an unreadable object will return an error indicating the object is not found. +** *can_read* grants read-only access to the record. Attempting to update or delete the record returns an error. *can_read* does not allow a reader to see any permission grants on the object except the object's owner_uuid and the reader's own permissions. +** *can_write* permits changes to the record (but not permission links). *can_write* permits the user to delete the object. *can_write* also implies *can_read*. +** *can_manage* permits the user to read, create, update and delete permission links whose @head_uuid@ is this object's @uuid@. *can_manage* also implies *can_write* and *can_read*. +h2. Ownership -Each API transaction (read, write, create, etc.) is done on behalf of a person. - -* An end user, via a web app -* The owner of an installed app - -A user (person) is permitted to act on an object if there is a path (series of permission Links) from the acting user to the object in which - -* Every intervening object is a Group, and -* Every intervening permission Link allows the current action - -Special case: A permission path can also include intervening User objects if the links _to_ the Users are "can_manage" links. - -Each object has exactly one _owner_, which can be either a User or a Group. - -* If the owner of X is A, then A is permitted to do any action on X. - -h3. Tokens - -An authorization token is issued at a user's request, and supplied to an API client using some suitable mechanism (_e.g._, cookie or application config file for a web app; environment variable or .rc-file for a CLI app). - -A user can have multiple valid tokens at a given time. At the user's option, a token can be restricted to a combination of - -* API client program -* time interval -* transaction type - -h3. System pseudo-user - -A privileged user account exists for the use of built-in Arvados system components. This user manages system-wide shared objects which can't really be "owned" by any particular user, like - -* Jobs and job steps (because a given job can be "wanted" by multiple users) -* Provenance metadata (because no user should be able to modify this directly) -* Storage metadata like -** redundancy verified as N× at time Y -** contents of collections A and B are identical - -The system pseudo-user's uuid is @{siteprefix}-tpzed-000000000000000@. - -h2. Example scenarios - -h3. 1. Private objects - -Alfred stores 3 data Collections in Keep and adds them to a new Group. - -The Collections and the Group can only be seen by Alfred, administrators, and the system user. - -The data in the Collections can only be retrieved by Alfred, administrators, and the system user. - -h3. 2. Public objects - -George creates a "PGP public data" Group, and grants "read" permission to all users. - -* ...by adding a Link: "All users" Group _can_read_→ "PGP public data" Group - -George stores 4 data Collections in Keep and adds them to the "PGP public data" Group +* All Arvados objects have an @owner_uuid@ field. Valid uuid types for @owner_uuid@ are "User" and "Group". +* The User or Group specified by @owner_uuid@ has *can_manage* permission on the object. +** This permission is one way: A User or Group's @owner_uuid@ being equal to @X@ does not imply any permission for that User/Group to read, write, or manage an object whose @uuid@ is equal to @X@. +* Applications should represent each object as belonging to, or being "inside", the Group/User referenced by its @owner_uuid@. +** A "project" is a subtype of Group that is treated as a "Project" in Workbench, and as a directory by @arv-mount@. +** A "role" is a subtype of Group that is treated in Workbench as a group of users who have permissions in common (typically an organizational group). +* To change the @owner_uuid@ field, it is necessary to have @can_write@ permission on both the current owner and the new owner. -* ...by adding a Link: Group _can_read_→ Collection +h2(#links). Permission links -Anyone who can connect to Arvados can log in with a Google/OpenID account and read the data. +A link object with -h3. 3. Group-managed objects +* @owner_uuid@ of the system user. +* @link_class@ "permission" +* @name@ one of *can_read*, *can_write* or *can_manage* +* @head_uuid@ of some Arvados object +* @tail_uuid@ of a User or Group -Three lab members are working together on a project. All Specimens, Links, Jobs, etc. can be modified by any of the three lab members. _Other_ lab members, who are not working on this project, can view but not modify these objects. +grants the @name@ permission for @tail_uuid@ accessing @head_uuid@ -h3. 4. Group-level administrator +* If a User has *can_manage* permission on some object, this grants permission to read, create, update and delete permission links where the @head_uuid@ is the object under management. -The Ashton Lab administrator, Alison, manages user accounts within her lab. She can enable and disable accounts, and exercise any permission that her lab members have. +h3. Transitive permissions -George has read-only access to the same set of accounts. This lets him see things like user activity and resource usage reports, without worrying about accidentally messing up anyone's data. +Permissions can be obtained indirectly through Groups. +* If a User X *can_read* Group A, and Group A *can_read* Object B, then User X *can_read* Object B. +* Permissions are narrowed to the least powerful permission on the path. +** If User X *can_write* Group A, and Group A *can_read* Object B, then User X *can_read* Object B. +** If User X *can_read* Group A, and Group A *can_write* Object B, then User X *can_read* Object B. -table(table table-bordered table-condensed). -|Tail |Permission |Head |Effect| -|Group: Ashton Lab Admin|can_manage |User: Lab Member 1 |Lab member 1 is in this administrative group| -|Group: Ashton Lab Admin|can_manage |User: Lab Member 2 |Lab member 2 is in this administrative group| -|Group: Ashton Lab Admin|can_manage |User: Lab Member 3 |Lab member 3 is in this administrative group| -|Group: Ashton Lab Admin|can_manage |User: Alison |Alison is in this administrative group| -|Group: Ashton Lab Admin|can_manage |User: George |George is in this administrative group| -|Alison |can_manage |Group: Ashton Lab Admin |Alison can do everything the above lab members can do| -|George |can_read |Group: Ashton Lab Admin |George can read everything the above lab members can read| +h2. Group Membership -h3. 5. Segregated roles +Group membership is determined by whether the group has *can_read* permission on an object. If a group G *can_read* an object A, then we say A is a member of G. -Granwyth, at the Hulatberi Lab, sets up a Factory Robot which uses a hosted Arvados site to do work for the Hulatberi Lab. +For some kinds of groups, like roles, it is natural for users who are members of a group to also have *can_manage* permission on the group, i.e., G *can_read* A and A *can_manage* G ("A can do anything G can do"). However, this is not necessary: A can be a member of a group while being unable to even read it. -Frank uploads a data Collection using Factory Robot's upload interface. Factory Robot sets data owner to Hulatberi Lab. +h2. Special cases -Factory Robot processes the data using a pipeline. +* Log table objects are additionally readable based on whether the User has *can_read* permission on @object_uuid@ (User can access log history about objects it can read). To retain the integrity of the log, the log table should deny all update or delete operations. +* Permission links where @tail_uuid@ is a User permit @can_read@ on the link by that user. (User can discover her own permission grants.) +* *can_read* on a Collection grants permission to read the blocks that make up the collection (API server returns signed blocks) +* If User or Group X *can_FOO* Group A, and Group A *can_manage* User B, then X *can_FOO* _everything that User B can_FOO_. -Factory Robot grants permission for anyone in the Ingeborg Lab (a Hulateberi Lab customer) to read the output of the pipeline, as well as the pipeline invocation details. (Say, Ingeborg and Jill.) +h2(#system). System user and group -During and after processing, all members of the Hulatberi Lab (_e.g._, Mike) can inspect pipeline progress, read source/intermediate/output data, and modify objects. +A privileged user account exists for the use by internal Arvados components. This user manages system objects which should not be "owned" by any particular user. The system user uuid is @{siteprefix}-tpzed-000000000000000@. -Possible encoding: +h2. Anoymous user and group -table(table table-bordered table-condensed). -|Tail |Permission |Head |Effect| -|Frank |(none) | |Factory Robot uses only its own credentials during upload| -|Granwyth |can_manage |User: Factory Robot |can revoke tokens, view activity... (needed?)| -|Granwyth |can_manage |Group: Hulatberi Lab |can grant group-write permission to Factory Robot| -|Factory Robot |can_write |Group: Hulatberi Lab |can add data, pipelines, jobs, etc. to the Lab group| -|Mike |can_write |Group: Hulatberi Lab |can edit/annotate/delete objects that belong to the Lab| +An Arvado site may be configued to allow users to browse resources without requiring a log in. In this case, permissions for non-logged-in users are associated with the "anonymous" user. To make objects visible to the public, they can be shared with the "anonymous" group. The anonymous user uuid is @{siteprefix}-tpzed-anonymouspublic@. The anonymous group uuid is @{siteprefix}-j7d0g-anonymouspublic@. -h3. Actions governed by permissions +h2. Example -table(table table-bordered table-condensed). -|_Action_|_Permissions needed_| -|Retrieve data from Keep|can_read (system-wide?)| -|Store data in Keep|can_write (system-wide?)| -|Add a Collection to Arvados|can_write (system-wide?)| -|Run a job|can_run (system-wide?)| -|See progress/result of a job|can_read (on job)| -|Give group permissions to a user/group|can_manage (on group)| -|Revoke group permissions from a user/group|can_manage (on group)| -|Change owner of an object|can_manage (on object)| -|Add an object to a group|can_write (on group)| +!{{site.baseurl}}/images/Arvados_Permissions.svg! diff --git a/doc/api/requests.html.textile.liquid b/doc/api/requests.html.textile.liquid new file mode 100644 index 0000000000..e720395e29 --- /dev/null +++ b/doc/api/requests.html.textile.liquid @@ -0,0 +1,349 @@ +--- +layout: default +navsection: api +navmenu: Concepts +title: REST API syntax +... + +Arvados exposes a REST API using standard HTTP requests. + +h3. HTTP Method + +Use @GET@ to request individual resources or lists of resources. + +Use @POST@ to create new resources. + +Use @PUT@ to update an existing resource. + +Use @DELETE@ to remove an existing resource. + +As a special case, a @POST@ with the query parameter @_method=GET@ will be treated as a GET request. This makes it possible to issue @GET@ requests where the query string exceeds the maximum request URI length, by putting the query string in the body of the request. + +h3. Request URI + +The URI portion of the request identifies the specific resource to operate on. For example, operations on "collections":{{site.baseurl}}/api/methods/collections.html use the @https://{{ site.arvados_api_host }}/arvados/v1/collections@ request URI prefix. + +h3. Authorization header + +Every request must include an API token. This identifies the user making the request for the purposes of access control. In addition, tokens may be further "restricted in scope":{{site.baseurl}}/api/methods/api_client_authorizations.html#scope to only access certain API endpoints. + +API requests must provide the API token using the @Authorization@ header in the following format: + +
+$ curl -v -H "Authorization: OAuth2 xxxxapitokenxxxx" https://192.168.5.2:8000/arvados/v1/collections
+> GET /arvados/v1/collections HTTP/1.1
+> ...
+> Authorization: OAuth2 xxxxapitokenxxxx
+> ...
+
+ +h3. Parameters + +Request parameters may be provided in one of two ways. They may be provided in the "query" section of request URI, or they may be provided in the body of the request with application/x-www-form-urlencoded encoding. If parameters are provided in both places, their values will be merged. Parameter names must be unique. If a parameter appears multiple times, the behavior is undefined. + +Structured and nested parameter values must be provided as urlencoded JSON. + +h3. Result + +Results are returned JSON-encoded in the response body. + +h3. Errors + +If a request cannot be fulfilled, the API will return 4xx or 5xx HTTP status code. Be aware that the API server may return a 404 (Not Found) status for resources that exist but for which the client does not have read access. The API will also return an error record: + +table(table table-bordered table-condensed). +|*Parameter name*|*Value*|*Description*| +|errors|array|An array of one or more error messages| +|error_token|string|a unique identifier used to correlate the error in the API server logs| + +h2. Examples + +h3. Create a new record + +
+$ curl -v -X POST --data-urlencode 'collection={"name":"empty collection"}' -H "Authorization: OAuth2 oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr" https://192.168.5.2:8000/arvados/v1/collections | jq .
+> POST /arvados/v1/collections HTTP/1.1
+> User-Agent: curl/7.38.0
+> Host: 192.168.5.2:8000
+> Accept: */*
+> Authorization: OAuth2 oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr
+> Content-Length: 54
+> Content-Type: application/x-www-form-urlencoded
+>
+} [data not shown]
+< HTTP/1.1 200 OK
+< Content-Type: application/json; charset=utf-8
+< Transfer-Encoding: chunked
+< Connection: keep-alive
+< Status: 200 OK
+< Access-Control-Allow-Origin: *
+< Access-Control-Allow-Methods: GET, HEAD, PUT, POST, DELETE
+< Access-Control-Allow-Headers: Authorization
+< Access-Control-Max-Age: 86486400
+< X-UA-Compatible: IE=Edge,chrome=1
+< ETag: "2ec9ef5151c1f7a1486ad169c33ae462"
+< Cache-Control: max-age=0, private, must-revalidate
+< Set-Cookie: _server_session=BAh7BkkiD3Nlc3Npb25faWQGOgZFVEkiJTIwMjQ1NTE5YmEwMzU1MGZkMTBmYmY1YzllY2ZiMjFlBjsAVA%3D%3D--653bc9c20899d48ee8523e18d9a4c1cde0702577; path=/; HttpOnly
+< X-Request-Id: 56aa10bc49097f3b44d3ed946bf0e61e
+< X-Runtime: 0.049951
+< X-Powered-By: Phusion Passenger 4.0.41
+< Date: Fri, 28 Oct 2016 19:20:09 GMT
+< Server: nginx/1.4.7 + Phusion Passenger 4.0.41
+<
+{
+  "href": "/collections/962eh-4zz18-m1ma0mxxfg3mbcc",
+  "kind": "arvados#collection",
+  "etag": "c5ifrv1ox2tu6alb559ymtkb7",
+  "uuid": "962eh-4zz18-m1ma0mxxfg3mbcc",
+  "owner_uuid": "962eh-tpzed-000000000000000",
+  "created_at": "2016-10-28T19:20:09.320771531Z",
+  "modified_by_client_uuid": "962eh-ozdt8-lm5x8emraox8epg",
+  "modified_by_user_uuid": "962eh-tpzed-000000000000000",
+  "modified_at": "2016-10-28T19:20:09.319661000Z",
+  "name": "empty collection",
+  "description": null,
+  "properties": {},
+  "portable_data_hash": "d41d8cd98f00b204e9800998ecf8427e+0",
+  "manifest_text": "",
+  "replication_desired": null,
+  "replication_confirmed": null,
+  "replication_confirmed_at": null,
+  "expires_at": null
+}
+
+ +h3. Delete a record + +
+$ curl -X DELETE -v -H "Authorization: OAuth2 oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr" https://192.168.5.2:8000/arvados/v1/collections/962eh-4zz18-m1ma0mxxfg3mbcc | jq .
+> DELETE /arvados/v1/collections/962eh-4zz18-m1ma0mxxfg3mbcc HTTP/1.1
+> User-Agent: curl/7.38.0
+> Host: 192.168.5.2:8000
+> Accept: */*
+> Authorization: OAuth2 oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr
+>
+< HTTP/1.1 200 OK
+< Content-Type: application/json; charset=utf-8
+< Transfer-Encoding: chunked
+< Connection: keep-alive
+< Status: 200 OK
+< Access-Control-Allow-Origin: *
+< Access-Control-Allow-Methods: GET, HEAD, PUT, POST, DELETE
+< Access-Control-Allow-Headers: Authorization
+< Access-Control-Max-Age: 86486400
+< X-UA-Compatible: IE=Edge,chrome=1
+< ETag: "1e8f72802cf1a6d0a5c4a1ebbfcc46a9"
+< Cache-Control: max-age=0, private, must-revalidate
+< Set-Cookie: _server_session=BAh7BkkiD3Nlc3Npb25faWQGOgZFVEkiJTc2NDYyY2M0NTNlNmU3M2Y2M2E3YmFiMWQ1MTEyZGZkBjsAVA%3D%3D--d28c7dd640bd24e2b12f01e77088072138dcf145; path=/; HttpOnly
+< X-Request-Id: e66fd3ab825bdb87301f5456161fb641
+< X-Runtime: 0.028788
+< X-Powered-By: Phusion Passenger 4.0.41
+< Date: Fri, 28 Oct 2016 19:33:31 GMT
+< Server: nginx/1.4.7 + Phusion Passenger 4.0.41
+<
+{
+  "href": "/collections/962eh-4zz18-m1ma0mxxfg3mbcc",
+  "kind": "arvados#collection",
+  "etag": "c5ifrv1ox2tu6alb559ymtkb7",
+  "uuid": "962eh-4zz18-m1ma0mxxfg3mbcc",
+  "owner_uuid": "962eh-tpzed-000000000000000",
+  "created_at": "2016-10-28T19:20:09.320771000Z",
+  "modified_by_client_uuid": "962eh-ozdt8-lm5x8emraox8epg",
+  "modified_by_user_uuid": "962eh-tpzed-000000000000000",
+  "modified_at": "2016-10-28T19:20:09.319661000Z",
+  "name": "empty collection",
+  "description": null,
+  "properties": {},
+  "portable_data_hash": "d41d8cd98f00b204e9800998ecf8427e+0",
+  "manifest_text": "",
+  "replication_desired": null,
+  "replication_confirmed": null,
+  "replication_confirmed_at": null,
+  "expires_at": null
+}
+
+ +h3. Get a specific record + +
+$ curl -v -H "Authorization: OAuth2 oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr" https://192.168.5.2:8000/arvados/v1/collections/962eh-4zz18-xi32mpz2621o8km | jq .
+> GET /arvados/v1/collections/962eh-4zz18-xi32mpz2621o8km HTTP/1.1
+> User-Agent: curl/7.38.0
+> Host: 192.168.5.2:8000
+> Accept: */*
+> Authorization: OAuth2 oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr
+>
+< HTTP/1.1 200 OK
+< Content-Type: application/json; charset=utf-8
+< Transfer-Encoding: chunked
+< Connection: keep-alive
+< Status: 200 OK
+< Access-Control-Allow-Origin: *
+< Access-Control-Allow-Methods: GET, HEAD, PUT, POST, DELETE
+< Access-Control-Allow-Headers: Authorization
+< Access-Control-Max-Age: 86486400
+< X-UA-Compatible: IE=Edge,chrome=1
+< ETag: "fec2ddf433a352e5a2b5d356abd6d3d4"
+< Cache-Control: max-age=0, private, must-revalidate
+< X-Request-Id: 40b447507ff202ae9a0b0b3e0ebe98da
+< X-Runtime: 0.011404
+< X-Powered-By: Phusion Passenger 4.0.41
+< Date: Fri, 28 Oct 2016 18:59:09 GMT
+< Server: nginx/1.4.7 + Phusion Passenger 4.0.41
+<
+{
+  "href": "/collections/962eh-4zz18-xi32mpz2621o8km",
+  "kind": "arvados#collection",
+  "etag": "3mmn0s9e1z5s5opfofmtb9k8p",
+  "uuid": "962eh-4zz18-xi32mpz2621o8km",
+  "owner_uuid": "962eh-tpzed-000000000000000",
+  "created_at": "2016-10-27T14:47:43.792587000Z",
+  "modified_by_client_uuid": "962eh-ozdt8-lm5x8emraox8epg",
+  "modified_by_user_uuid": "962eh-tpzed-000000000000000",
+  "modified_at": "2016-10-27T14:47:43.792166000Z",
+  "name": "Saved at 2016-10-27 14:47:43 UTC by peter@debian",
+  "description": null,
+  "properties": {},
+  "portable_data_hash": "93a45073511646a5c3e2f4953fcf6f61+116",
+  "manifest_text": ". eff999f3b5158331eb44a9a93e3b36e1+67108864+Aad3839bea88bce22cbfe71cf4943de7dab3ea52a@5826180f db141bfd11f7da60dce9e5ee85a988b8+34038725+Ae8f48913fed782cbe463e0499ab37697ee06a2f8@5826180f 0:101147589:rna.SRR948778.bam\n",
+  "replication_desired": null,
+  "replication_confirmed": null,
+  "replication_confirmed_at": null,
+  "expires_at": null
+}
+
+ +h3. List records and filter by date + +(Note, return result is truncated). + +
+$ curl -v -G --data-urlencode 'filters=[["created_at",">","2016-11-08T21:38:24.124834000Z"]]' -H "Authorization: OAuth2 oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr" https://192.168.5.2:8000/arvados/v1/collections | jq .
+> GET /arvados/v1/collections?filters=%5B%5B%22uuid%22%2C%20%22%3D%22%2C%20%22962eh-4zz18-xi32mpz2621o8km%22%5D%5D HTTP/1.1
+> User-Agent: curl/7.38.0
+> Host: 192.168.5.2:8000
+> Accept: */*
+> Authorization: OAuth2 oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr
+>
+< HTTP/1.1 200 OK
+< Content-Type: application/json; charset=utf-8
+< Transfer-Encoding: chunked
+< Connection: keep-alive
+< Status: 200 OK
+< Access-Control-Allow-Origin: *
+< Access-Control-Allow-Methods: GET, HEAD, PUT, POST, DELETE
+< Access-Control-Allow-Headers: Authorization
+< Access-Control-Max-Age: 86486400
+< X-UA-Compatible: IE=Edge,chrome=1
+< ETag: "76345ef24952f073acc3a0c550241d4e"
+< Cache-Control: max-age=0, private, must-revalidate
+< X-Request-Id: d34b8ede4ffc707d8ed172dc2f47ff5e
+< X-Runtime: 0.012727
+< X-Powered-By: Phusion Passenger 4.0.41
+< Date: Fri, 28 Oct 2016 19:08:52 GMT
+< Server: nginx/1.4.7 + Phusion Passenger 4.0.41
+<
+{
+  "kind": "arvados#collectionList",
+  "etag": "",
+  "self_link": "",
+  "offset": 0,
+  "limit": 100,
+  "items": [
+    {
+      "href": "/collections/962eh-4zz18-ybggo9im899vv60",
+      "kind": "arvados#collection",
+      "etag": "bvgrrsg63zsenb9wnpnp0nsgl",
+      "uuid": "962eh-4zz18-ybggo9im899vv60",
+      "owner_uuid": "962eh-tpzed-000000000000000",
+      "created_at": "2016-11-08T21:47:36.937106000Z",
+      "modified_by_client_uuid": null,
+      "modified_by_user_uuid": "962eh-tpzed-000000000000000",
+      "modified_at": "2016-11-08T21:47:36.936625000Z",
+      "name": "Log from cwl-runner job 962eh-8i9sb-45jww0k15fi5ldd",
+      "description": null,
+      "properties": {},
+      "portable_data_hash": "a7820b94717eff86229927565fedbd72+85",
+      "replication_desired": null,
+      "replication_confirmed": null,
+      "replication_confirmed_at": null,
+      "expires_at": null
+    },
+   ...
+    {
+      "href": "/collections/962eh-4zz18-37i1tfl5de5ild9",
+      "kind": "arvados#collection",
+      "etag": "2fa07dx52lux8wa1loehwyrc5",
+      "uuid": "962eh-4zz18-37i1tfl5de5ild9",
+      "owner_uuid": "962eh-tpzed-000000000000000",
+      "created_at": "2016-11-08T21:38:46.717798000Z",
+      "modified_by_client_uuid": null,
+      "modified_by_user_uuid": "962eh-tpzed-000000000000000",
+      "modified_at": "2016-11-08T21:38:46.717409000Z",
+      "name": null,
+      "description": null,
+      "properties": {},
+      "portable_data_hash": "9d43d4c8328640446f6e252cda584e7e+54",
+      "replication_desired": null,
+      "replication_confirmed": null,
+      "replication_confirmed_at": null,
+      "expires_at": null
+    }
+  ],
+  "items_available": 99
+}
+
+ +h3. Update a field + +
+$ curl -v -X PUT --data-urlencode 'collection={"name":"rna.SRR948778.bam"}' -H "Authorization: OAuth2 oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr" https://192.168.5.2:8000/arvados/v1/collections/962eh-4zz18-xi32mpz2621o8km | jq .
+> PUT /arvados/v1/collections/962eh-4zz18-xi32mpz2621o8km HTTP/1.1
+> User-Agent: curl/7.38.0
+> Host: 192.168.5.2:8000
+> Accept: */*
+> Authorization: OAuth2 oz0os4nyudswvglxhdlnrgnuelxptmj7qu7dpwvyz3g9ocqtr
+> Content-Length: 53
+> Content-Type: application/x-www-form-urlencoded
+>
+} [data not shown]
+< HTTP/1.1 200 OK
+< Content-Type: application/json; charset=utf-8
+< Transfer-Encoding: chunked
+< Connection: keep-alive
+< Status: 200 OK
+< Access-Control-Allow-Origin: *
+< Access-Control-Allow-Methods: GET, HEAD, PUT, POST, DELETE
+< Access-Control-Allow-Headers: Authorization
+< Access-Control-Max-Age: 86486400
+< X-UA-Compatible: IE=Edge,chrome=1
+< ETag: "fbb50d2847426eab793e3fcf346ca9eb"
+< Cache-Control: max-age=0, private, must-revalidate
+< Set-Cookie: _server_session=BAh7BkkiD3Nlc3Npb25faWQGOgZFVEkiJWI3NjFjMzVjMGI5OGExYmNjZDg0ZTg5MjZhMzcwMDE1BjsAVA%3D%3D--0e005d71fad15cb366e47361c38474b7447ba155; path=/; HttpOnly
+< X-Request-Id: 76d3cb3c0995af6133b0a73a64f57354
+< X-Runtime: 0.030756
+< X-Powered-By: Phusion Passenger 4.0.41
+< Date: Fri, 28 Oct 2016 19:15:16 GMT
+< Server: nginx/1.4.7 + Phusion Passenger 4.0.41
+<
+{
+  "href": "/collections/962eh-4zz18-xi32mpz2621o8km",
+  "kind": "arvados#collection",
+  "etag": "51509hhxo9qqjxqewnoz1b7og",
+  "uuid": "962eh-4zz18-xi32mpz2621o8km",
+  "owner_uuid": "962eh-tpzed-000000000000000",
+  "created_at": "2016-10-27T14:47:43.792587000Z",
+  "modified_by_client_uuid": "962eh-ozdt8-lm5x8emraox8epg",
+  "modified_by_user_uuid": "962eh-tpzed-000000000000000",
+  "modified_at": "2016-10-28T19:15:16.137814000Z",
+  "name": "rna.SRR948778.bam",
+  "description": null,
+  "properties": {},
+  "portable_data_hash": "93a45073511646a5c3e2f4953fcf6f61+116",
+  "manifest_text": ". eff999f3b5158331eb44a9a93e3b36e1+67108864+Acca57af82cc18c5dfa47bdfd16e335fccd09dfa5@582618c4 db141bfd11f7da60dce9e5ee85a988b8+34038725+A7764f122f41f92c2d5bde1852fcdd1bea5f8bd78@582618c4 0:101147589:rna.SRR948778.bam\n",
+  "replication_desired": null,
+  "replication_confirmed": null,
+  "replication_confirmed_at": null,
+  "expires_at": null
+}
+
diff --git a/doc/api/resources.html.textile.liquid b/doc/api/resources.html.textile.liquid index a93b4ac114..7b36f113bf 100644 --- a/doc/api/resources.html.textile.liquid +++ b/doc/api/resources.html.textile.liquid @@ -2,48 +2,38 @@ layout: default navsection: api navmenu: Concepts -title: Resources +title: Common resource fields ... - - This page describes the common attributes of Arvados resources. -The list of resource types is on the "front page of the API Reference":./. - -h2. Object IDs - -Object IDs are alphanumeric strings, unique across all installations (each installation has a unique prefix to prevent collisions). - -h2(#resource). Attributes of resources +h2(#resource). Resource table(table table-bordered table-condensed). -|*Attribute*|*Type*|*Description*|*Example*| -|uuid|string|universally unique object identifier|@mk2qn-4zz18-w3anr2hk2wgfpuo@| +|_. Attribute |_. Type |_. Description |_. Example| +|uuid|string|universally unique object identifier, set on @create@|@mk2qn-4zz18-w3anr2hk2wgfpuo@| +|owner_uuid|string|UUID of owner (must be a User or Group), set on @create@, controls who may access the resource, ownership may be changed explicitly with @update@, see "permission model":{{site.baseurl}}/api/permission-model.html for details.|@mk2qn-tpzed-a4lcehql0dv2u25@| +|created_at|datetime|When resource was created, set on @create@|@2013-01-21T22:17:39Z@| +|modified_by_client_uuid|string|API client software which most recently modified the resource, set on @create@ and @update@|@mk2qn-ozdt8-vq8l5qkzj7pr7h7@| +|modified_by_user_uuid|string|Authenticated user, on whose behalf the client was acting when modifying the resource, set on @create@ and @update@|@mk2qn-tpzed-a4lcehql0dv2u25@| +|modified_at|datetime|When resource was last modified, set on @create@ and @update@|@2013-01-25T22:29:32Z@| |href|string|a URL that can be used to address this resource|| |kind|string|@arvados#{resource_type}@|@arvados#collection@| |etag|string|The ETag[1] of the resource|@1xlmizzjq7wro3dlb2dirf505@| -|self_link|string||| -|owner_uuid|string|UUID of owner (typically User or Project)|@mk2qn-tpzed-a4lcehql0dv2u25@| -|created_at|datetime|When resource was created|@2013-01-21T22:17:39Z@| -|modified_by_client_uuid|string|API client software which most recently modified the resource|@mk2qn-ozdt8-vq8l5qkzj7pr7h7@| -|modified_by_user_uuid|string|Authenticated user, on whose behalf the client was acting when modifying the resource|@mk2qn-tpzed-a4lcehql0dv2u25@| -|modified_at|datetime|When resource was last modified|@2013-01-25T22:29:32Z@| -h2(#resourceList). Attributes of resource lists +h2. Object UUID -table(table table-bordered table-condensed). -|*Attribute*|*Type*|*Description*|*Example*| -|kind|string|@arvados#{resource_type}List@|@arvados#projectList@| -|etag|string|The ETag[1] of the resource list|@cd3o1wi9sf934saajykawrz2e@| -|self_link|string||| -|next_page_token|string||| -|next_link|string||| -|items[]|list|List of resources|| +Each object is assigned a UUID. This has the format @aaaaa-bbbbb-ccccccccccccccc@. + +# The first field (@aaaaa@ in the example) is the site prefix. This is unique to a specific Arvados installation. +# The second field (@bbbbb@ in the example) is the object type. +# The third field (@ccccccccccccccc@ in the example) uniquely identifies the object. +h2. Timestamps + +All Arvados timestamps follow ISO 8601 datetime format with fractional seconds (microsecond precision). All timestamps are UTC. Date format: @YYYY-mm-ddTHH:MM:SS.SSSSZ@ example date: @2016-11-08T21:38:24.124834000Z@. h2. ETags fn1. Each response includes an ETag, a string which changes when the resource changes. Clients can use this to check whether a resource has changed since they last retrieved it. If a previous ETag is provided along with a request, and the resource has not changed since, the server may return a "not modified" response. - diff --git a/doc/api/schema/ApiClient.html.textile.liquid b/doc/api/schema/ApiClient.html.textile.liquid deleted file mode 100644 index 0cda1af150..0000000000 --- a/doc/api/schema/ApiClient.html.textile.liquid +++ /dev/null @@ -1,24 +0,0 @@ ---- -layout: default -navsection: api -navmenu: Schema -title: ApiClient - -... - - -An **ApiClient** represents a client program that can issue requests to the API server. - -h2. Methods - -See "api_clients":{{site.baseurl}}/api/methods/api_clients.html - -h2. Resource - -Each ApiClient has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html: - -table(table table-bordered table-condensed). -|_. Attribute|_. Type|_. Description|_. Example| -|name|string||| -|url_prefix|string||| -|is_trusted|boolean|Trusted by users to handle their API tokens (ApiClientAuthorizations).|| diff --git a/doc/api/schema/ApiClientAuthorization.html.textile.liquid b/doc/api/schema/ApiClientAuthorization.html.textile.liquid deleted file mode 100644 index dc9614aeaa..0000000000 --- a/doc/api/schema/ApiClientAuthorization.html.textile.liquid +++ /dev/null @@ -1,29 +0,0 @@ ---- -layout: default -navsection: api -navmenu: Schema -title: ApiClientAuthorization - -... - -An **ApiClientAuthorization** represents an API client's authorization to make API requests on a user's behalf. - -h2. Methods - -See "api_client_authorizations":{{site.baseurl}}/api/methods/api_client_authorizations.html - -h2. Resource - -An ApiClientAuthorization is not a generic Arvados resource. The full list of properties that belong to an ApiClientAuthorization is: - -table(table table-bordered table-condensed). -|_. Attribute|_. Type|_. Description|_. Example| -|api_token|string||| -|api_client_id|integer||| -|user_id|integer||| -|created_by_ip_address|string||| -|last_used_by_ip_address|string||| -|last_used_at|datetime||| -|expires_at|datetime||| -|default_owner_uuid|string||| -|scopes|array||| diff --git a/doc/api/schema/AuthorizedKey.html.textile.liquid b/doc/api/schema/AuthorizedKey.html.textile.liquid deleted file mode 100644 index d9f4354d83..0000000000 --- a/doc/api/schema/AuthorizedKey.html.textile.liquid +++ /dev/null @@ -1,24 +0,0 @@ ---- -layout: default -navsection: api -navmenu: Schema -title: AuthorizedKey -... - -An **AuthorizedKey** represents the public part of an SSH authentication key which can be used to authorize transactions on behalf of the user. - -h2. Methods - -See "authorized_keys":{{site.baseurl}}/api/methods/authorized_keys.html - -h2. Resource - -Each AuthorizedKey has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html: - -table(table table-bordered table-condensed). -|_. Attribute|_. Type|_. Description|_. Example| -|name|string||| -|key_type|string||| -|authorized_user_uuid|string||| -|public_key|text||| -|expires_at|datetime||| diff --git a/doc/api/schema/Collection.html.textile.liquid b/doc/api/schema/Collection.html.textile.liquid deleted file mode 100644 index 9aa783cbf5..0000000000 --- a/doc/api/schema/Collection.html.textile.liquid +++ /dev/null @@ -1,39 +0,0 @@ ---- -layout: default -navsection: api -navmenu: Schema -title: Collection - -... - -Note: This resource concerns indexing, usage accounting, and integrity checks for data stored in Arvados. Reading and writing the data _per se_ is achieved by the "Keep":{{site.baseurl}}/user/tutorials/tutorial-keep.html storage system. - -h2. Methods - -See "collections":{{site.baseurl}}/api/methods/collections.html - -h3. Conditions of creating a Collection - -The @uuid@ and @manifest_text@ attributes must be provided when creating a Collection. The cryptographic digest of the supplied @manifest_text@ must match the supplied @uuid@. - -h3. Side effects of creating a Collection - -Referenced data can be protected from garbage collection. See the section about "resources" links on the "Links":Link.html page. - -Data can be shared with other users via the Arvados permission model. - -Clients can request checks of data integrity and storage redundancy. - -h2. Resource - -Each collection has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html: - -table(table table-bordered table-condensed). -|_. Attribute|_. Type|_. Description|_. Example| -|name|string||| -|description|text||| -|portable_data_hash|string||| -|manifest_text|text||| -|replication_desired|number|Minimum storage replication level desired for each data block referenced by this collection. A value of @null@ signifies that the site default replication level (typically 2) is desired.|@2@| -|replication_confirmed|number|Replication level most recently confirmed by the storage system. This field is null when a collection is first created, and is reset to null when the manifest_text changes in a way that introduces a new data block. An integer value indicates the replication level of the _least replicated_ data block in the collection.|@2@, null| -|replication_confirmed_at|datetime|When replication_confirmed was confirmed. If replication_confirmed is null, this field is also null.|| diff --git a/doc/api/schema/Container.html.textile.liquid b/doc/api/schema/Container.html.textile.liquid deleted file mode 100644 index d29d420829..0000000000 --- a/doc/api/schema/Container.html.textile.liquid +++ /dev/null @@ -1,59 +0,0 @@ ---- -layout: default -navsection: api -navmenu: Schema -title: Container - -... - -A Container: -* Precisely describes the environment in which a Crunch2 process should run. For example, git trees, data collections, and docker images are stored as content addresses. This makes it possible to reason about the difference between two processes, and to replay a process at a different time and place. -* Container records are created by the system to fulfill container requests. - -h2. Methods - -See "containers":{{site.baseurl}}/api/methods/containers.html - -h2. Resource - -Each Container offers the following attributes, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html: - -table(table table-bordered table-condensed). -|_. Attribute|_. Type|_. Description|_. Notes| -|state|string|The allowed states are "Queued", "Locked", "Running", "Cancelled" and "Complete".|See "Container states":#container_states for more details.| -|started_at|datetime|When this container started running.|Null if container has not yet started.| -|finished_at|datetime|When this container finished.|Null if container has not yet finished.| -|log|string|Portable data hash of the collection containing logs from a completed container run.|Null if the container is not yet finished.| -|environment|hash|Environment variables and values that should be set in the container environment (@docker run --env@). This augments and (when conflicts exist) overrides environment variables given in the image's Dockerfile.|Must be equal to a ContainerRequest's environment in order to satisfy the ContainerRequest.| -|cwd|string|Initial working directory.|Must be equal to a ContainerRequest's cwd in order to satisfy the ContainerRequest| -|command|array of strings|Command to execute.| Must be equal to a ContainerRequest's command in order to satisfy the ContainerRequest.| -|output_path|string|Path to a directory or file inside the container that should be preserved as this container's output when it finishes.|Must be equal to a ContainerRequest's output_path in order to satisfy the ContainerRequest.| -|mounts|hash|Must contain the same keys as the ContainerRequest being satisfied. Each value must be within the range of values described in the ContainerRequest at the time the Container is assigned to the ContainerRequest.|See "Mount types":#mount_types for more details.| -|runtime_constraints|hash|Compute resources, and access to the outside world, that are / were available to the container. -Generally this will contain additional keys that are not present in any corresponding ContainerRequests: for example, even if no ContainerRequests specified constraints on the number of CPU cores, the number of cores actually used will be recorded here.|e.g., -
{
-  "ram":12000000000,
-  "vcpus":2,
-  "API":true
-}
See "Runtime constraints":#runtime_constraints for more details.| -|output|string|Portable data hash of the output collection.|Null if the container is not yet finished.| -|container_image|string|Portable data hash of a collection containing the docker image used to run the container.|| -|progress|number|A number between 0.0 and 1.0 describing the fraction of work done.|| -|priority|integer|Priority assigned by the system, taking into account the priorities of all associated ContainerRequests.|| -|exit_code|integer|Process exit code.|Null if state!="Complete"| -|auth_uuid|string|UUID of a token to be passed into the container itself, used to access Keep-backed mounts, etc.|Null if state∉{"Locked","Running"}| -|locked_by_uuid|string|UUID of a token, indicating which dispatch process changed state to Locked. If null, any token can be used to lock. If not null, only the indicated token can modify this container.|Null if state∉{"Locked","Running"}| - -h2(#container_states). Container states - -table(table table-bordered table-condensed). -|_. State|_. Sgnificance|_. Allowed next| -|Queued|Waiting for a dispatcher to lock it and try to run the container.|Locked, Cancelled| -|Locked|A dispatcher has "taken" the container and is allocating resources for it. The container has not started yet.|Queued, Running, Cancelled| -|Running|Resources have been allocated and the contained process has been started (or is about to start). Crunch-run _must_ set state to Running _before_ there is any possibility that user code will run in the container.|Complete, Cancelled| -|Complete|Container was running, and the contained process/command has exited.|-| -|Cancelled|The container did not run long enough to produce an exit code. This includes cases where the container didn't even start, cases where the container was interrupted/killed before it exited by itself (e.g., priority changed to 0), and cases where some problem prevented the system from capturing the contained process's exit status (exit code and output).|-| - -h2(#mount_types). {% include 'mount_types' %} - -h2(#runtime_constraints). {% include 'container_runtime_constraints' %} diff --git a/doc/api/schema/ContainerRequest.html.textile.liquid b/doc/api/schema/ContainerRequest.html.textile.liquid deleted file mode 100644 index 48c624a69e..0000000000 --- a/doc/api/schema/ContainerRequest.html.textile.liquid +++ /dev/null @@ -1,70 +0,0 @@ ---- -layout: default -navsection: api -navmenu: Schema -title: ContainerRequest - -... - -A ContainerRequest: -* Is a client's expression of interest in knowing the outcome of a computational process. -* The system is responsible for finding suitable containers and assigning them to container_requests. -* The client's description of the ContainerRequest is less precise than of a Container: a ContainerRequest describes container constraints which can have different interpretations over time. For example, a ContainerRequest with a {"kind":"git_tree","commit_range":"abc123..master",...} mount might be satisfiable by any of several different source trees, and this set of satisfying source trees can change when the repository's "master" branch is updated. - -h2. Methods - -See "container_requests":{{site.baseurl}}/api/methods/container_requests.html - -h2. Resource - -Each ContainerRequest offers the following attributes, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html: - -All attributes are optional, unless otherwise marked as required. - -table(table table-bordered table-condensed). -|_. Attribute|_. Type|_. Description|_. Notes| -|name|string|The name of the container_request.|| -|description|string|The description of the container_request.|| -|properties|hash|Client-defined structured data that does not affect how the container is run.|| -|state|string|The allowed states are "Uncommitted", "Committed", and "Final".|Once a request is Committed, the only attributes that can be modified are priority, container_uuid, and container_count_max. A request in the "Final" state cannot have any of its functional parts modified (i.e., only name, description, and properties fields can be modified).| -|requesting_container_uuid|string|The uuid of the parent container that created this container_request, if any. Represents a process tree.|The priority of this container_request is inherited from the parent container, if the parent container is cancelled, this container_request will be cancelled as well.| -|container_uuid|string|The uuid of the container that satisfies this container_request. The system will find and reuse any preexisting Container that matches this ContainerRequest's criteria. See "Container reuse":#container_reuse for more details.|Currently, container reuse is the default behavior and a mechanism to skip reuse is not supported.| -|container_count_max|integer|Maximum number of containers to start, i.e., the maximum number of "attempts" to be made.|| -|mounts|hash|Objects to attach to the container's filesystem and stdin/stdout.|See "Mount types":#mount_types for more details.| -|runtime_constraints|hash|Restrict the container's access to compute resources and the outside world.|Required when in "Committed" state. e.g.,
{
-  "ram":12000000000,
-  "vcpus":2,
-  "API":true
-}
See "Runtime constraints":#runtime_constraints for more details.| -|container_image|string|Portable data hash of a collection containing the docker image to run the container.|Required.| -|environment|hash|Environment variables and values that should be set in the container environment (@docker run --env@). This augments and (when conflicts exist) overrides environment variables given in the image's Dockerfile.|| -|cwd|string|Initial working directory, given as an absolute path (in the container) or a path relative to the WORKDIR given in the image's Dockerfile.|Required.| -|command|array of strings|Command to execute in the container.|Required. e.g., @["echo","hello"]@| -|output_path|string|Path to a directory or file inside the container that should be preserved as container's output when it finishes. This path must be, or be inside, one of the mount targets. For best performance, point output_path to a writable collection mount.|Required.| -|priority|integer|Higher value means spend more resources on this container_request, i.e., go ahead of other queued containers, bring up more nodes etc.|Priority 0 means a container should not be run on behalf of this request. Clients are expected to submit ContainerRequests with zero priority in order to prevew the container that will be used to satisfy it. Priority can be null if and only if state!="Committed".| -|expires_at|datetime|After this time, priority is considered to be zero.|Not yet implemented.| -|filters|string|Additional constraints for satisfying the container_request, given in the same form as the filters parameter accepted by the container_requests.list API.|| - -h2(#mount_types). {% include 'mount_types' %} - -h2(#runtime_constraints). {% include 'container_runtime_constraints' %} - -h2(#container_reuse). Container reuse - -When a ContainerRequest is "Committed", the system will try to find and reuse any preexisting Container with the same exact command, cwd, environment, output_path, container_image, mounts, and runtime_constraints as this ContainerRequest. The serialized fields environment, mounts and runtime_constraints are sorted to facilitate comparison. - -The system will use the following scheme to determine which Container to consider for reuse: A Container with the same exact command, cwd, environment, output_path, container_image, mounts, and runtime_constraints as this ContainerRequest and, -* The oldest successfully finished container, i.e., in state "Complete" with exit_code of 0. If matching containers with different outputs are found, the system will forgo reusing any of these finished containers and instead look for suitable containers in other states -* The oldest "Running" container with the highest progress, i.e., the container that is most likely to finish first -* The oldest "Locked" container with the highest priority, i.e., the container that is most likely to start first -* The oldest "Queued" container with the highest priority, i.e, the container that is most likely to start first - -{% include 'notebox_begin' %} -Currently, container reuse is the default behavior and a mechanism to skip reuse is not supported. -{% include 'notebox_end' %} - -h2(#cancel_container). Canceling a ContainerRequest - -A ContainerRequest may be canceled by setting it's priority to 0, using an update call. - -When a ContainerRequest is canceled, it will still reflect the state of the Container it is associated with via the container_uuid attribute. If that Container is being reused by any other container_requests that are still active, i.e., not yet canceled, that Container may continue to run or be scheduled to run by the system in future. However, if no other container_requests are using that Contianer, then the Container will get canceled as well. diff --git a/doc/api/schema/Group.html.textile.liquid b/doc/api/schema/Group.html.textile.liquid deleted file mode 100644 index 2bf67eb6ff..0000000000 --- a/doc/api/schema/Group.html.textile.liquid +++ /dev/null @@ -1,25 +0,0 @@ ---- -layout: default -navsection: api -navmenu: Schema -title: Group - -... - -A **Group** represents a set of objects. Groups allow you to organize content, define user roles, and apply permissions to sets of objects. - -h2. Methods - -See "groups":{{site.baseurl}}/api/methods/groups.html - -h2. Resource - -Each Group has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html: - -table(table table-bordered table-condensed). -|_. Attribute|_. Type|_. Description|_. Example| -|name|string||| -|group_class|string|Type of group. This does not affect behavior, but determines how the group is presented in the user interface. For example, @project@ indicates that the group should be displayed by Workbench and arv-mount as a project for organizing and naming objects.|@"project"@ -null| -|description|text||| -|writable_by|array|List of UUID strings identifying Users and other Groups that have write permission for this Group. Only users who are allowed to administer the Group will receive a full list. Other users will receive a partial list that includes the Group's owner_uuid and (if applicable) their own user UUID.|| diff --git a/doc/api/schema/Human.html.textile.liquid b/doc/api/schema/Human.html.textile.liquid deleted file mode 100644 index 361e619659..0000000000 --- a/doc/api/schema/Human.html.textile.liquid +++ /dev/null @@ -1,19 +0,0 @@ ---- -layout: default -navsection: api -navmenu: Schema -title: Human - -... - -h2. Methods - -See "humans":{{site.baseurl}}/api/methods/humans.html - -h2. Resource - -Each Human has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html: - -table(table table-bordered table-condensed). -|_. Attribute|_. Type|_. Description|_. Example| -|properties|hash||| diff --git a/doc/api/schema/Job.html.textile.liquid b/doc/api/schema/Job.html.textile.liquid deleted file mode 100644 index 58b6a51cb4..0000000000 --- a/doc/api/schema/Job.html.textile.liquid +++ /dev/null @@ -1,69 +0,0 @@ ---- -layout: default -navsection: api -navmenu: Schema -title: Job - -... - -Applications submit compute jobs when: -* Provenance is important, i.e., it is worth recording how the output was produced; or -* Computation time is significant; or -* The job management features are convenient (failure detection/recovery, regression testing, etc). - -h2. Methods - -See "jobs":{{site.baseurl}}/api/methods/jobs.html - -h2. Resource - -Each job has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html: - -table(table table-bordered table-condensed). -|_. Attribute|_. Type|_. Description|_. Notes| -|script|string|The filename of the job script.|This program will be invoked by Crunch for each job task. It is given as a path to an executable file, relative to the @/crunch_scripts@ directory in the Git tree specified by the _repository_ and _script_version_ attributes.| -|script_parameters|hash|The input parameters for the job.|Conventionally, one of the parameters is called @"input"@. Typically, some parameter values are collection UUIDs. Ultimately, though, the significance of parameters is left entirely up to the script itself.| -|repository|string|Git repository name or URL.|Source of the repository where the given script_version is to be found. This can be given as the name of a locally hosted repository, or as a publicly accessible URL starting with @git://@, @http://@, or @https://@. -Examples: -@yourusername/yourrepo@ -@https://github.com/curoverse/arvados.git@| -|script_version|string|Git commit|During a **create** transaction, this is the Git branch, tag, or hash supplied by the client. Before the job starts, Arvados updates it to the full 40-character SHA-1 hash of the commit used by the job. -See "Specifying Git versions":#script_version below for more detail about acceptable ways to specify a commit.| -|cancelled_by_client_uuid|string|API client ID|Is null if job has not been cancelled| -|cancelled_by_user_uuid|string|Authenticated user ID|Is null if job has not been cancelled| -|cancelled_at|datetime|When job was cancelled|Is null if job has not been cancelled| -|started_at|datetime|When job started running|Is null if job has not [yet] started| -|finished_at|datetime|When job finished running|Is null if job has not [yet] finished| -|running|boolean|Whether the job is running|| -|success|boolean|Whether the job indicated successful completion|Is null if job has not finished| -|is_locked_by_uuid|string|UUID of the user who has locked this job|Is null if job is not locked. The system user locks the job when starting the job, in order to prevent job attributes from being altered.| -|node_uuids|array|List of UUID strings for node objects that have been assigned to this job|| -|log|string|Collection UUID|Is null if the job has not finished. After the job runs, the given collection contains a text file with log messages provided by the @arv-crunch-job@ task scheduler as well as the standard error streams provided by the task processes.| -|tasks_summary|hash|Summary of task completion states.|Example: @{"done":0,"running":4,"todo":2,"failed":0}@| -|output|string|Collection UUID|Is null if the job has not finished.| -|nondeterministic|boolean|The job is expected to produce different results if run more than once.|If true, this job will not be considered as a candidate for automatic re-use when submitting subsequent identical jobs.| -|submit_id|string|Unique ID provided by client when job was submitted|Optional. This can be used by a client to make the "jobs.create":{{site.baseurl}}/api/methods/jobs.html#create method idempotent.| -|priority|string||| -|arvados_sdk_version|string|Git commit hash that specifies the SDK version to use from the Arvados repository|This is set by searching the Arvados repository for a match for the arvados_sdk_version runtime constraint.| -|docker_image_locator|string|Portable data hash of the collection that contains the Docker image to use|This is set by searching readable collections for a match for the docker_image runtime constraint.| -|runtime_constraints|hash|Constraints that must be satisfied by the job/task scheduler in order to run the job.|See below.| -|components|hash|Name and uuid pairs representing the child work units of this job. The uuids can be of different object types.|Example components hash: @{"name1": "zzzzz-8i9sb-xyz...", "name2": "zzzzz-d1hrv-xyz...",}@| - -h3(#script_version). Specifying Git versions - -The script_version attribute and arvados_sdk_version runtime constraint are typically given as a branch, tag, or commit hash, but there are many more ways to specify a Git commit. The "specifying revisions" section of the "gitrevisions manual page":http://git-scm.com/docs/gitrevisions.html has a definitive list. Arvados accepts Git versions in any format listed there that names a single commit (not a tree, a blob, or a range of commits). However, some kinds of names can be expected to resolve differently in Arvados than they do in your local repository. For example, HEAD@{1} refers to the local reflog, and @origin/master@ typically refers to a remote branch: neither is likely to work as desired if given as a Git version. - -h3. Runtime constraints - -table(table table-bordered table-condensed). -|_. Key|_. Type|_. Description|_. Implemented| -|arvados_sdk_version|string|The Git version of the SDKs to use from the Arvados git repository. See "Specifying Git versions":#script_version for more detail about acceptable ways to specify a commit. If you use this, you must also specify a @docker_image@ constraint (see below). In order to install the Python SDK successfully, Crunch must be able to find and run virtualenv inside the container.|✓| -|docker_image|string|The Docker image that this Job needs to run. If specified, Crunch will create a Docker container from this image, and run the Job's script inside that. The Keep mount and work directories will be available as volumes inside this container. The image must be uploaded to Arvados using @arv keep docker@. You may specify the image in any format that Docker accepts, such as @arvados/jobs@, @debian:latest@, or the Docker image id. Alternatively, you may specify the portable data hash of the image Collection.|✓| -|min_nodes|integer||✓| -|max_nodes|integer||| -|min_cores_per_node|integer|Require that each node assigned to this Job have the specified number of CPU cores|✓| -|min_ram_mb_per_node|integer|Require that each node assigned to this Job have the specified amount of real memory (in MiB)|✓| -|min_scratch_mb_per_node|integer|Require that each node assigned to this Job have the specified amount of scratch storage available (in MiB)|✓| -|max_tasks_per_node|integer|Maximum simultaneous tasks on a single node|✓| -|keep_cache_mb_per_task|integer|Size of file data buffer for per-task Keep directory ($TASK_KEEPMOUNT), in MiB. Default is 256 MiB. Increase this to reduce cache thrashing in situtations such as accessing multiple large (64+ MiB) files at the same time, or accessing different parts of a large file at the same time.|✓| -|min_ram_per_task|integer|Minimum real memory (KiB) per task|| diff --git a/doc/api/schema/JobTask.html.textile.liquid b/doc/api/schema/JobTask.html.textile.liquid deleted file mode 100644 index fbd4343f88..0000000000 --- a/doc/api/schema/JobTask.html.textile.liquid +++ /dev/null @@ -1,47 +0,0 @@ ---- -layout: default -navsection: api -navmenu: Schema -title: JobTask - -... - -A Job Task is a well defined independently-computable portion of a "Job":Job.html. - -Job tasks are created two ways: -* When a job starts, it is seeded with a job task with @sequence=0@ and an empty @parameters{}@ list. -* Job task A can create additional job tasks B, C, D, which will belong to the same job. Tasks B, C, D will not be performed until job task A is complete. If job task A fails, tasks B, C, D will be deleted. - -Job tasks have particular update semantics: -* Progress reporting: A job task should only be PATCHed by a worker process which has been dispatched to work on that task and is reporting progress or completion status — and by the job manager itself. -* Completion: When a job task process terminates, the task is considered complete only if its most recent @PATCH@ transaction had @progress=1.0@ and @success=true@. -* Temporary failure: If a job task process terminates without updating @success@ to @true@ or @false@, it is assumed that the task failed but is worth re-attempting (at a different time, on a different node, etc). - - -h2. Methods - -See "job_tasks":{{site.baseurl}}/api/methods/job_tasks.html - -h2. Resource - -Each JobTask has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html: - -table(table table-bordered table-condensed). -|_. Attribute|_. Type|_. Description|_. Example| -|sequence|integer|Execution sequence. -A step cannot be run until all steps with lower sequence numbers have completed. -Job steps with the same sequence number can be run in any order.|| -|parameters|hash||| -|output|text||| -|progress|float||| -|success|boolean|Is null if the task has neither completed successfully nor failed permanently.|| - -The following attributes should not be updated by anyone other than the job manager: - -table(table table-bordered table-condensed). -|_. Attribute|_. Type|_. Description|_. Notes| -|qsequence|integer|Order of arrival|0-based| -|job_uuid|string||| -|created_by_job_task_uuid|string||| - - diff --git a/doc/api/schema/KeepDisk.html.textile.liquid b/doc/api/schema/KeepDisk.html.textile.liquid deleted file mode 100644 index c128c3e865..0000000000 --- a/doc/api/schema/KeepDisk.html.textile.liquid +++ /dev/null @@ -1,31 +0,0 @@ ---- -layout: default -navsection: api -navmenu: Schema -title: KeepDisk - -... - -A **KeepDisk** is a filesystem volume used by a Keep storage server to store data blocks. - -h2. Methods - -See "keep_disks":{{site.baseurl}}/api/methods/keep_disks.html - -h2. Resource - -Each KeepDisk has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html: - -table(table table-bordered table-condensed). -|_. Attribute|_. Type|_. Description|_. Example| -|ping_secret|string||| -|node_uuid|string||| -|filesystem_uuid|string||| -|bytes_total|integer||| -|bytes_free|integer||| -|is_readable|boolean||| -|is_writable|boolean||| -|last_read_at|datetime||| -|last_write_at|datetime||| -|last_ping_at|datetime||| -|keep_service_uuid|string||| diff --git a/doc/api/schema/KeepService.html.textile.liquid b/doc/api/schema/KeepService.html.textile.liquid deleted file mode 100644 index ac1d9742df..0000000000 --- a/doc/api/schema/KeepService.html.textile.liquid +++ /dev/null @@ -1,24 +0,0 @@ ---- -layout: default -navsection: api -navmenu: Schema -title: KeepService - -... - -A **KeepService** is a service endpoint that supports the Keep protocol. - -h2. Methods - -See "keep_services":{{site.baseurl}}/api/methods/keep_services.html - -h2. Resource - -Each KeepService has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html: - -table(table table-bordered table-condensed). -|_. Attribute|_. Type|_. Description|_. Example| -|service_host|string||| -|service_port|integer||| -|service_ssl_flag|boolean||| -|service_type|string||| \ No newline at end of file diff --git a/doc/api/schema/Link.html.textile.liquid b/doc/api/schema/Link.html.textile.liquid deleted file mode 100644 index 4abfdbc249..0000000000 --- a/doc/api/schema/Link.html.textile.liquid +++ /dev/null @@ -1,83 +0,0 @@ ---- -layout: default -navsection: api -navmenu: Schema -title: Link - -... - -**Links** describe relationships between Arvados objects, and from objects to primitives. - -Links are directional: each metadata object has a tail (the "subject" being described), class, name, properties, and head (the "object" that describes the "subject"). A Link may describe a relationship between two objects in an Arvados database: e.g. a _permission_ link between a User and a Group defines the permissions that User has to read or modify the Group. Other Links simply represent metadata for a single object, e.g. the _identifier_ Link, in which the _name_ property represents a human-readable identifier for the object at the link's head. - -For links that don't make sense to share between API clients, a _link_class_ that begins with @client@ (like @client.my_app_id@ or @client.my_app_id.anythinghere@) should be used. - -h2. Methods - -See "links":{{site.baseurl}}/api/methods/links.html - -h2. Resource - -Each link has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html: - -table(table table-bordered table-condensed). -|_. Attribute|_. Type|_. Description| -|tail_uuid|string|Object UUID at the tail (start, source, origin) of this link| -|link_class|string|Class (see below)| -|name|string|Link type (see below)| -|head_uuid|string|Object UUID at the head (end, destination, target) of this link| -|properties|hash|Additional information, expressed as a key→value hash. Key: string. Value: string, number, array, or hash.| - -h2. Link classes - -Some classes are pre-defined by convention and have standard meanings attached to names. - -h3. provenance - -table(table table-bordered table-condensed). -|_. tail_type→head_type|_. name→head_uuid {properties}|_. Notes| -|→Collection |provided → _collection uuid_ -{url→http://example.com/foo.tgz, retrieved_at→1352616640.000}|| -|Job→Collection |provided → _collection uuid_|| -|Specimen→Collection|provided → _collection uuid_|| -|Human→Specimen |provided → _specimen uuid_|| -|Human→Collection |provided → _collection uuid_|| - -h3. permission - -table(table table-bordered table-condensed). -|_. tail_type→head_type|_. name→head_uuid {properties}|_. Notes| -|User→Group |{white-space:nowrap}. can_manage → _group uuid_|The User can read, write, and control permissions on the Group itself, every object owned by the Group, and every object on which the Group has _can_manage_ permission.| -|User→Group |can_read → _group uuid_ |The User can retrieve the Group itself and every object that is readable by the Group.| -|User→Job|can_write → _job uuid_ |The User can read and update the Job. (This works for all objects, not just jobs.)| -|User→Job|can_manage → _job uuid_ |The User can read, update, and change permissions for the Job. (This works for all objects, not just jobs.)| -|Group→Job|can_manage → _job uuid_ |Anyone with _can_manage_ permission on the Group can also read, update, and change permissions for the Job. Anyone with _can_read_ permission on the Group can read the Job. (This works for all objects, not just jobs.)| - -h3. resources - -table(table table-bordered table-condensed). -|_. tail_type→head_type|_. name→head_uuid {properties}|_. Notes| -|User→Collection|wants → _collection uuid_ |Determines whether data can be deleted| -|User→Job |wants → _job uuid_ |Determines whether a job can be cancelled| - -h3. tag - -A **tag** link describes an object using an unparsed plain text string. Tags can be used to annotate objects that are not editable, like collections and objects shared as read-only. - -table(table table-bordered table-condensed). -|_. tail_type→head_type|_. name→head_uuid {properties}| -|→Collection | _tag name_ → _collection uuid_| -|→Job | _tag name_ → _job uuid_| - -h3. human_trait - -table(table table-bordered table-condensed). -|_. tail_type→head_type|_. name→head_uuid {properties}|_. Notes| -|Human→Trait |measured → _trait uuid_ {value→1.83, unit→metre, measured_at→1352616640.000}|| - -h3. identifier - -table(table table-bordered table-condensed). -|_. tail_type→head_type|_. name→head_uuid {properties}|_. Notes| -|→Human |hu123456 → _human uuid_|| - diff --git a/doc/api/schema/Log.html.textile.liquid b/doc/api/schema/Log.html.textile.liquid deleted file mode 100644 index 425246a221..0000000000 --- a/doc/api/schema/Log.html.textile.liquid +++ /dev/null @@ -1,33 +0,0 @@ ---- -layout: default -navsection: api -navmenu: Schema -title: Log - -... - -**Log** objects record events that occur in an Arvados cluster. Both user-written pipelines and the Arvados system itself may generate Log events. - -h2. Methods - -See "logs":{{site.baseurl}}/api/methods/logs.html - -h2. Creation - -Any user may create Log entries for any event they find useful. User-generated Logs have no intrinsic meaning to other users or to the Arvados system itself; it is up to each user to choose appropriate log event types and summaries for their project. - -h3. System Logs - -Arvados uses Logs to record creation, deletion, and updates of other Arvados resources. - -h2. Resource - -Each Log has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html: - -table(table table-bordered table-condensed). -|_. Attribute|_. Type|_. Description|_. Example| -|object_uuid|string||| -|event_at|datetime||| -|event_type|string|A user-defined category or type for this event.|@LOGIN@| -|summary|text||| -|properties|hash||| diff --git a/doc/api/schema/Node.html.textile.liquid b/doc/api/schema/Node.html.textile.liquid deleted file mode 100644 index ff9e882040..0000000000 --- a/doc/api/schema/Node.html.textile.liquid +++ /dev/null @@ -1,28 +0,0 @@ ---- -layout: default -navsection: api -navmenu: Schema -title: Node - -... - -A **Node** represents a host that can be used to run Crunch job tasks. - -h2. Methods - -See "nodes":{{site.baseurl}}/api/methods/nodes.html - -h2. Resource - -Each Node has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html: - -table(table table-bordered table-condensed). -|_. Attribute|_. Type|_. Description|_. Example| -|slot_number|integer||| -|hostname|string||| -|domain|string||| -|ip_address|string||| -|job_uuid|string|The UUID of the job that this node is assigned to work on. If you do not have permission to read the job, this will be null.|| -|first_ping_at|datetime||| -|last_ping_at|datetime||| -|info|hash||| diff --git a/doc/api/schema/PipelineInstance.html.textile.liquid b/doc/api/schema/PipelineInstance.html.textile.liquid deleted file mode 100644 index 75c788533b..0000000000 --- a/doc/api/schema/PipelineInstance.html.textile.liquid +++ /dev/null @@ -1,26 +0,0 @@ ---- -layout: default -navsection: api -navmenu: Schema -title: PipelineInstance - -... - -A **PipelineInstance** is the act or record of applying a pipeline template to a specific set of inputs; generally, a pipeline instance refers to a set of jobs that have been run to satisfy the pipeline components. - -h2. Methods - -See "pipeline_instances":{{site.baseurl}}/api/methods/pipeline_instances.html - -h2. Resource - -Each PipelineInstance has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html: - -table(table table-bordered table-condensed). -|_. Attribute|_. Type|_. Description|_. Example| -|pipeline_template_uuid|string||| -|name|string||| -|components|hash||| -|success|boolean||| -|active|boolean||| -|properties|Hash||| diff --git a/doc/api/schema/PipelineTemplate.html.textile.liquid b/doc/api/schema/PipelineTemplate.html.textile.liquid deleted file mode 100644 index 444960a6ea..0000000000 --- a/doc/api/schema/PipelineTemplate.html.textile.liquid +++ /dev/null @@ -1,161 +0,0 @@ ---- -layout: default -navsection: api -navmenu: Schema -title: PipelineTemplate -... - -The pipeline template consists of "name" and "components". - -table(table table-bordered table-condensed). -|_. Attribute |_. Type |_. Accepted values |_. Required|_. Description| -|name |string |any |yes |The human-readable name of the pipeline template.| -|components |object |JSON object containing job submission objects|yes |The component jobs that make up the pipeline, with the component name as the key. | - -h3. Components - -The components field of the pipeline template is a JSON object which describes the individual steps that make up the pipeline. Each component is an Arvados job submission. "Parameters for job submissions are described on the job method page.":{{site.baseurl}}/api/methods/jobs.html#create In addition, a component can have the following parameters: - -table(table table-bordered table-condensed). -|_. Attribute |_. Type |_. Accepted values |_. Required|_. Description| -|output_name |string or boolean|string or false |no |If a string is provided, use this name for the output collection of this component. If the value is false, do not create a permanent output collection (an temporary intermediate collection will still be created). If not provided, a default name will be assigned to the output.| - -h3. Script parameters - -When used in a pipeline, each parameter in the 'script_parameters' attribute of a component job can specify that the input parameter must be supplied by the user, or the input parameter should be linked to the output of another component. To do this, the value of the parameter should be JSON object containing one of the following attributes: - -table(table table-bordered table-condensed). -|_. Attribute |_. Type |_. Accepted values |_. Description| -|default |any |any |The default value for this parameter.| -|required |boolean |true or false |Specifies whether the parameter is required to have a value or not.| -|dataclass |string |One of 'Collection', 'File' [1], 'number', or 'text' |Data type of this parameter.| -|search_for |string |any string |Substring to use as a default search string when choosing inputs.| -|output_of |string |the name of another component in the pipeline |Specifies that the value of this parameter should be set to the 'output' attribute of the job that corresponds to the specified component.| -|title |string |any string |User friendly title to display when choosing parameter values| -|description |string |any string |Extended text description for describing expected/valid values for the script parameter| -|link_name |string |any string |User friendly name to display for the parameter value instead of the actual parameter value| - -The 'output_of' parameter is especially important, as this is how components are actually linked together to form a pipeline. Component jobs that depend on the output of other components do not run until the parent job completes and has produced output. If the parent job fails, the entire pipeline fails. - -fn1. The 'File' type refers to a specific file within a Keep collection in the form 'collection_hash/filename', for example '887cd41e9c613463eab2f0d885c6dd96+83/bob.txt'. - -The 'search_for' parameter is meaningful only when input dataclass of type Collection or File is used. If a value is provided, this will be preloaded into the input data chooser dialog in Workbench. For example, if your input dataclass is a File and you are interested in a certain filename extention, you can preconfigure it in this attribute. - -h3. Examples - -This is a pipeline named "Filter MD5 hash values" with two components, "do_hash" and "filter". The "input" script parameter of the "do_hash" component is required to be filled in by the user, and the expected data type is "Collection". This also specifies that the "input" script parameter of the "filter" component is the output of "do_hash", so "filter" will not run until "do_hash" completes successfully. When the pipeline runs, past jobs that meet the criteria described above may be substituted for either or both components to avoid redundant computation. - -
-{
-  "name": "Filter MD5 hash values",
-  "components": {
-    "do_hash": {
-      "script": "hash.py",
-      "repository": "you/you",
-      "script_version": "master",
-      "script_parameters": {
-        "input": {
-          "required": true,
-          "dataclass": "Collection",
-          "search_for": ".fastq.gz",
-          "title":"Please select a fastq file"
-        }
-      },
-    },
-    "filter": {
-      "script": "0-filter.py",
-      "repository": "you/you",
-      "script_version": "master",
-      "script_parameters": {
-        "input": {
-          "output_of": "do_hash"
-        }
-      },
-    }
-  }
-}
-
 - -This pipeline consists of three components. The components "thing1" and "thing2" both depend on "cat_in_the_hat". Once the "cat_in_the_hat" job is complete, both "thing1" and "thing2" can run in parallel, because they do not depend on each other. - -
-{
-  "name": "Wreck the house",
-  "components": {
-    "cat_in_the_hat": {
-      "script": "cat.py",
-      "repository": "you/you",
-      "script_version": "master",
-      "script_parameters": { }
-    },
-    "thing1": {
-      "script": "thing1.py",
-      "repository": "you/you",
-      "script_version": "master",
-      "script_parameters": {
-        "input": {
-          "output_of": "cat_in_the_hat"
-        }
-      },
-    },
-    "thing2": {
-      "script": "thing2.py",
-      "repository": "you/you",
-      "script_version": "master",
-      "script_parameters": {
-        "input": {
-          "output_of": "cat_in_the_hat"
-        }
-      },
-    },
-  }
-}
-
 - -This pipeline consists of three components. The component "cleanup" depends on "thing1" and "thing2". Both "thing1" and "thing2" are started immediately and can run in parallel, because they do not depend on each other, but "cleanup" cannot begin until both "thing1" and "thing2" have completed. - -
-{
-  "name": "Clean the house",
-  "components": {
-    "thing1": {
-      "script": "thing1.py",
-      "repository": "you/you",
-      "script_version": "master",
-      "script_parameters": { }
-    },
-    "thing2": {
-      "script": "thing2.py",
-      "repository": "you/you",
-      "script_version": "master",
-      "script_parameters": { }
-    },
-    "cleanup": {
-      "script": "cleanup.py",
-      "repository": "you/you",
-      "script_version": "master",
-      "script_parameters": {
-        "mess1": {
-          "output_of": "thing1"
-        },
-        "mess2": {
-          "output_of": "thing2"
-        }
-      }
-    }
-  }
-}
-
 - -h2. Methods - -See "pipeline_templates":{{site.baseurl}}/api/methods/pipeline_templates.html - -h2. Resource - -Each PipelineTemplate has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html: - -table(table table-bordered table-condensed). -|_. Attribute|_. Type|_. Description|_. Example| -|name|string||| -|components|hash||| diff --git a/doc/api/schema/Repository.html.textile.liquid b/doc/api/schema/Repository.html.textile.liquid deleted file mode 100644 index 0f9b25ec2c..0000000000 --- a/doc/api/schema/Repository.html.textile.liquid +++ /dev/null @@ -1,25 +0,0 @@ ---- -layout: default -navsection: api -navmenu: Schema -title: Repository - -... - -A **Repository** represents a git repository hosted in an Arvados installation. - -h2. Methods - -See "repositories":{{site.baseurl}}/api/methods/repositories.html - -h2. Resource - -Each Repository has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html: - -table(table table-bordered table-condensed). -|_. Attribute|_. Type|_. Description|_. Example| -|name|string|The name of the repository on disk. Repository names must begin with a letter and contain only alphanumerics. Unless the repository is owned by the system user, the name must begin with the owner's username, then be separated from the base repository name with @/@. You may not create a repository that is owned by a user without a username.|@username/project1@| -|clone_urls|array|URLs from which the repository can be cloned. Read-only.|@["git@git.zzzzz.arvadosapi.com:foo/bar.git", - "https://git.zzzzz.arvadosapi.com/foo/bar.git"]@| -|fetch_url|string|URL suggested as a fetch-url in git config. Deprecated. Read-only.|| -|push_url|string|URL suggested as a push-url in git config. Deprecated. Read-only.|| diff --git a/doc/api/schema/Specimen.html.textile.liquid b/doc/api/schema/Specimen.html.textile.liquid deleted file mode 100644 index 1a7e483d55..0000000000 --- a/doc/api/schema/Specimen.html.textile.liquid +++ /dev/null @@ -1,22 +0,0 @@ ---- -layout: default -navsection: api -navmenu: Schema -title: Specimen - -... - -A **Specimen** represents a tissue sample or similar material obtained from a human that has some biomedical significance or interest. - -h2. Methods - -See "specimens":{{site.baseurl}}/api/methods/specimens.html - -h2. Resource - -Each Specimen has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html: - -table(table table-bordered table-condensed). -|_. Attribute|_. Type|_. Description|_. Example| -|material|string||| -|properties|hash||| diff --git a/doc/api/schema/Trait.html.textile.liquid b/doc/api/schema/Trait.html.textile.liquid deleted file mode 100644 index 80c74ab19e..0000000000 --- a/doc/api/schema/Trait.html.textile.liquid +++ /dev/null @@ -1,22 +0,0 @@ ---- -layout: default -navsection: api -navmenu: Schema -title: Trait - -... - -A **Trait** represents a measured or observed characteristic of a human. - -h2. Methods - -See "traits":{{site.baseurl}}/api/methods/traits.html - -h2. Resource - -Each Trait has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html: - -table(table table-bordered table-condensed). -|_. Attribute|_. Type|_. Description|_. Example| -|name|string||| -|properties|hash||| diff --git a/doc/api/schema/User.html.textile.liquid b/doc/api/schema/User.html.textile.liquid deleted file mode 100644 index 335b6c1097..0000000000 --- a/doc/api/schema/User.html.textile.liquid +++ /dev/null @@ -1,30 +0,0 @@ ---- -layout: default -navsection: api -navmenu: Schema -title: User - -... - -A **User** represents a person who interacts with Arvados via an ApiClient. - -h2. Methods - -See "users":{{site.baseurl}}/api/methods/users.html - -h2. Resource - -Each User has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html: - -table(table table-bordered table-condensed). -|_. Attribute|_. Type|_. Description|_. Example| -|email|string||| -|username|string|The username used for the user's git repositories and virtual machine logins. Usernames must start with a letter, and contain only alphanumerics. When a new user is created, a default username is set from their e-mail address. Only administrators may change the username.|| -|first_name|string||| -|last_name|string||| -|identity_url|string||| -|is_admin|boolean||| -|prefs|hash||| -|default_owner_uuid|string||| -|is_active|boolean||| -|writable_by|array|List of UUID strings identifying Groups and other Users that can modify this User object. This will include the user's owner_uuid and, for administrators and users requesting their own User object, the requesting user's UUID.|| diff --git a/doc/api/schema/VirtualMachine.html.textile.liquid b/doc/api/schema/VirtualMachine.html.textile.liquid deleted file mode 100644 index 1c6a4b68ab..0000000000 --- a/doc/api/schema/VirtualMachine.html.textile.liquid +++ /dev/null @@ -1,21 +0,0 @@ ---- -layout: default -navsection: api -navmenu: Schema -title: VirtualMachine - -... - -A **VirtualMachine** represents a network host, running within an Arvados installation, on which Arvados users are given login accounts. - -h2. Methods - -See "virtual_machines":{{site.baseurl}}/api/methods/virtual_machines.html - -h2. Resource - -Each VirtualMachine has, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html: - -table(table table-bordered table-condensed). -|_. Attribute|_. Type|_. Description|_. Example| -|hostname|string||| diff --git a/doc/api/schema/Workflow.html.textile.liquid b/doc/api/schema/Workflow.html.textile.liquid deleted file mode 100644 index 05cd998823..0000000000 --- a/doc/api/schema/Workflow.html.textile.liquid +++ /dev/null @@ -1,23 +0,0 @@ ---- -layout: default -navsection: api -navmenu: Schema -title: Workflow - -... - -A *Workflow* is a definition of work to be performed by a Crunch2 process. It defines the steps and inputs for the process. - -h2. Methods - -See "workflows":{{site.baseurl}}/api/methods/workflows.html - -h2. Resource - -Each Workflow offers the following optional attributes, in addition to the usual "attributes of Arvados resources":{{site.baseurl}}/api/resources.html: - -table(table table-bordered table-condensed). -|_. Attribute|_. Type|_. Description|_. Example| -|name|string|If not specified, will be set to any "name" from the "definition" attribute.|| -|description|string|If not specified, will be set to any "description" from the "definition" attribute.|| -|definition|string|A "Common Workflow Language" document.|Visit "Common Workflow Language":http://www.commonwl.org/ for details.| diff --git a/doc/api/storage.html.textile.liquid b/doc/api/storage.html.textile.liquid new file mode 100644 index 0000000000..868e5cc170 --- /dev/null +++ b/doc/api/storage.html.textile.liquid @@ -0,0 +1,170 @@ +--- +layout: default +navsection: api +title: Storage in Keep +... + +Keep clients are applications such as @arv-get@, @arv-put@ and @arv-mount@ which store and retrieve data from Keep. In doing so, these programs interact with both the API server (which stores file metadata in form of Collection objects) and individual Keep servers (which store the actual data blocks). + +!{{site.baseurl}}/images/Keep_reading_writing_block.svg! + +h2. Storing a file + +# The client discovers keep servers (or proxies) using the @accessible@ method on "keep_services":{{site.baseurl}}/api/methods/keep_services.html +# Data is split into 64 MiB blocks and the MD5 hash is computed for each block. +# The client uploads each block to one or more Keep servers, based on the number of desired replicas. The priority order is determined using rendezvous hashing, described below. +# The Keep server returns a block locator (the MD5 sum of the block) and a "signed token" which the client can use as proof of knowledge for the block. +# The client constructs a @manifest@ which lists the blocks by MD5 hash and how to reassemble them into the original files. +# The client creates a "collection":{{site.baseurl}}/api/methods/collections.html and provides the @manifest_text@ +# The API server accepts the collection after validating the signed tokens (proof of knowledge) for each block. + +!{{site.baseurl}}/images/Keep_manifests.svg! + +h2. Fetching a file + +# The client requests a @collection@ object including @manifest_text@ from the APIs server +# The server adds "token signatures" to the @manifest_text@ and returns it to the client. +# The client discovers keep servers (or proxies) using the @accessible@ method on "keep_services":{{site.baseurl}}/api/methods/keep_services.html +# For each data block, the client chooses the highest priority server using rendezvous hashing, described below. +# The client sends the data block request to the keep server, along with the token signature from the API which proves to Keep servers that the client is permitted to read a given block. +# The server provides the block data after validating the token signature for the block (if the server does not have the block, it returns a 404 and the client tries the next highest priority server) + +!{{site.baseurl}}/images/Keep_rendezvous_hashing.svg! + +Each @keep_service@ resource has an assigned uuid. To determine priority assignments of blocks to servers, for each keep service compute the MD5 sum of the string concatenation of the block locator (hex-coded hash part only) and service uuid, then sort this list in descending order. Blocks are preferentially placed on servers with the highest weight. + +h2. Keep server API + +The Keep server is accessed via a simple HTTP REST API. + +*GET /blocklocator+size+A@token* + +Fetch the data block. Response returns block contents. If permission checking is enabled, requires a valid token hint. + +*PUT /blocklocator* + +Body: the block contents. Responds the block locator consisting of MD5 sum of the data, block size, and signed token hint. + +*POST /* + +Body: the block contents. Responds the block locator consisting of MD5 sum of the data, block size, and signed token hint. + +h2(#locator). Keep locator format + +BNF notation for a valid Keep locator string (with hints). For example @d41d8cd98f00b204e9800998ecf8427e+0+Z+Ada39a3ee5e6b4b0d3255bfef95601890afd80709@53bed294@ + +
+locator        ::= sized-digest hint*
+sized-digest   ::= digest size-hint
+digest         ::= <32 lowercase hexadecimal digits>
+size-hint      ::= "+" [0-9]+
+hint           ::= "+" hint-type hint-content
+hint-type      ::= [A-Z]+
+hint-content   ::= [A-Za-z0-9@_-]*
+sign-hint      ::= "+A" <40 lowercase hexadecimal digits> "@" sign-timestamp
+sign-timestamp ::= <8 lowercase hexadecimal digits>
+
+ +h3. Token signatures + +A token signature (sign-hint) provides proof-of-access for a data block. It is computed by taking a SHA1 HMAC of the blob signing token (a shared secret between the API server and keep servers), block digest, current API token, expiration timestamp, and blob signature TTL. + +When communicating with the Keep store to fetch a block, or the API server to create or update a collection, the service computes the expected token signature for each block and compares it to the token signature that was presented by the client. Keep clients receive valid block signatures when uploading a block to a keep store (getting back a signed token as proof of knowledge) or, from the API server, getting the manifest text of a collection on which the user has read permission. + +Security of a token signature is derived from the following characteristics: + +# Valid signatures can only be generated by entities that know the shared secret (the "blob signing token") +# A signature can only be used by an entity that also know the API token that was used to generate it. +# It expires after a set date (the expiration time, based on the "blob signature time-to-live (TTL)") + +h3. Regular expression to validate locator + +
+/^([0-9a-f]{32})\+([0-9]+)(\+[A-Z][-A-Za-z0-9@_]*)*$/
+
+ +h3. Valid locators + +table(table table-bordered table-condensed). +|@d41d8cd98f00b204e9800998ecf8427e+0@| +|@d41d8cd98f00b204e9800998ecf8427e+0+Z@| +|d41d8cd98f00b204e9800998ecf8427e+0+Z+Ada39a3ee5e6b4b0d3255bfef95601890afd80709@53bed294| + +h3. Invalid locators + +table(table table-bordered table-condensed). +||Why| +|@d41d8cd98f00b204e9800998ecf8427e@|No size hint| +|@d41d8cd98f00b204e9800998ecf8427e+Z+0@|Other hint before size hint| +|@d41d8cd98f00b204e9800998ecf8427e+0+0@|Multiple size hints| +|@d41d8cd98f00b204e9800998ecf8427e+0+z@|Hint does not start with uppercase letter| +|@d41d8cd98f00b204e9800998ecf8427e+0+Zfoo*bar@|Hint contains invalid character @*@| + +h2. Manifest v1 + +A manifest is utf-8 encoded text, consisting of zero or more newline-terminated streams. + +
+manifest       ::= stream*
+stream         ::= stream-name (" " locator)+ (" " file-segment)+ "\n"
+stream-name    ::= "." ("/" path-component)*
+path-component ::= +
+file-segment   ::= position ":" size ":" filename
+position       ::= [0-9]+
+size           ::= [0-9]+
+filename       ::= path-component ("/" path-component)*
+
+ +Notes: + +* The first token is the stream name, consisting of one or more path components, delimited by @"/"@. +** The first path component is always @"."@. +** No path component is empty. +** No path component following the first one can be "." or "..". +** The stream name never begins or ends with @"/"@. +* The next N tokens are "keep locators":#locator +** These describe the "data stream". By logically concatenating the blocks in the order that they appear, we can refer to "positions" in the data stream. +* File tokens come after the sequence of keep locators. +** A file token has three parts, delimited by @":"@: position, size, filename. +** Position and size are given in decimal +** The position is the position in the data stream +** The size is the count of bytes following the position in the data stream. A file size may cross multiple blocks in the data stream. +** Filename may contain @"/"@ characters, but must not start or end with @"/"@, and must not contain @"//"@. +** Filename components (delimited by @"/"@) must not be @"."@ or @".."@. +** There may be multiple file tokens. + +It is legal to have multiple file tokens in the manifest (possible across different streams) with the same combined path name @stream name + "/" + filename@. This must be interpreted as a concatenation of file content, in the order that the file tokens appear in the manifest. + +Spaces are represented by the escape sequence @\040@. Spaces in stream names and filenames must be translated when reading and writing manifests. A manifest may not contain TAB characters, nor other ASCII whitespace characters or control codes other than the spaces or newlines used as delimiters specified above. A manifest always ends with a newline -- except the empty (zero-length) string, which is a valid manifest. + +h3. Normalized manifest v1 + +A normalized manifest is a manifest that meets the following additional restrictions: + +* Streams are in alphanumeric order. +* Each stream name is unique within the manifest. +* Files within a stream are listed in alphanumeric order. +* Blocks within a stream are ordered based on order of file tokens of the stream. A given block is listed at most once in a stream. +* Filename must not contain @"/"@ (the stream name represents the path prefix) + +h3. Example manifests + +A manifest with four files in two directories: + +
+. 930625b054ce894ac40596c3f5a0d947+33 0:0:a 0:0:b 0:33:output.txt
+./c d41d8cd98f00b204e9800998ecf8427e+0 0:0:d
+
+ +The same manifest with permission signatures on each block: + +
+. 930625b054ce894ac40596c3f5a0d947+33+A1f27a35dd9af37191d63ad8eb8985624451e7b79@5835c8bc 0:0:a 0:0:b 0:33:output.txt
+./c d41d8cd98f00b204e9800998ecf8427e+0+A27117dcd30c013a6e85d6d74c9a50179a1446efa@5835c8bc 0:0:d
+
+ +A manifest containing a file consisting of multiple blocks and a space in the file name: + +
+. c449ed86671e4a34a8b8b9430850beba+67108864 09fcfea01c3a141b89dd0dcfa1b7768e+22534144 0:89643008:Docker\040image.tar
+
diff --git a/doc/api/tokens.html.textile.liquid b/doc/api/tokens.html.textile.liquid new file mode 100644 index 0000000000..d391d00312 --- /dev/null +++ b/doc/api/tokens.html.textile.liquid @@ -0,0 +1,63 @@ +--- +layout: default +navsection: api +title: API Authorization +... + +All requests to the API server must have an API token. API tokens can be issued by going though the login flow, or created via the API. At this time, only browser based applications can perform login from email/password. Command line applications and services must use an API token provided via the @ARVADOS_API_TOKEN@ environment variable or configuration file. + +h2. Browser login + +Browser based applications can perform log in via the following highlevel flow: + +# The web application presents a "login" link to @/login@ on the API server with a @return_to@ parameter provided in the query portion of the URL. For example @https://{{ site.arvados_api_host }}/login?return_to=XXX@ , where @return_to=XXX@ is the URL of the login page for the web application. +# The "login" link takes the browser to the login page (this may involve several redirects) +# The user logs in. API server authenticates the user and issues a new API token. +# The browser is redirected to the login page URL provided in @return_to=XXX@ with the addition of @?api_token=xxxxapitokenxxxx@. +# The web application gets the login request with the included authorization token. + +!{{site.baseurl}}/images/Session_Establishment.svg! + +The "browser authentication process is documented in detail on the Aravdos wiki.":https://dev.arvados.org/projects/arvados/wiki/Workbench_authentication_process + +h2. Creating tokens via the API + +The browser login method above issues a new token. Using that token, it is possible to make API calls to create additional tokens. To do so, use the @create@ method of the "API client authorizations":{{site.baseurl}}/api/methods/api_client_authorizations.html resource. + +h2. Trusted API clients + +The "api_clients":{{site.baseurl}}/api/methods/api_clients.html resource determines if web applications that have gone through the browser login flow may create or list API tokens. + +After the user has authenticated, but before an authorization token is issued and browser redirect sent (sending the browser back to the @return_to@ login page bearing @api_token@), the server strips the path and query portion from @return_to@ to get @url_prefix@. The @url_prefix@ is used to find or create an ApiClient object. The newly issued API client authorization (API token) is associated with this ApiClient object. + +API clients may be marked as "trusted" by making an API call to create or update an "api_clients":{{site.baseurl}}/api/methods/api_clients.html resource and set the @is_trusted@ flag to @true@. An authorization token associated with a "trusted" client is permitted to list authorization tokens on "API client authorizations":{{site.baseurl}}/api/methods/api_client_authorizations.html . + +A authorization token which is not associated with a trusted client may only use the @current@ method to query its own api_client_authorization object. The "untrusted" token is forbidden performing any other operations on API client authorizations, such as listing other authorizations or creating new authorizations. + +Authorization tokens which are not issued via the browser login flow (created directly via the API) will not have an associated api client. This means authorization tokens created via the API are always "untrusted". + +h2(#scopes). Scopes + +Scopes can restrict a token so it may only access certain resources. This is in addition to normal permission checks for the user associated with the token. + +Each entry in scopes consists of a @request_method@ and @request_path@, where the @request_method@ is a HTTP method (one of @GET@, @POST@, @PUT@ or @DELETE@) and @request_path@ is the request URI. A given request is permitted if it matches a scopes exactly, or the scope ends with @/@ and the request string is a prefix of the scope. + +As a special case, a scope of ["all"] allows all resources. + +h3. Scope examples + +A scope of @GET /arvados/v1/collections@ permits listing collections. + +* Requests with different methods, such as creating a new collection using @POST /arvados/v1/collections@, will be rejected. +* Requests to access other resources, such as @GET /arvados/v1/groups@, will be rejected. +* Be aware that requests for specific records, such as @GET /arvados/v1/collections/962eh-4zz18-xi32mpz2621o8km@ will also be rejected. This is because the scope @GET /arvados/v1/collections@ does not end in @/@ + +A scope of @GET /arvados/v1/collections/@ (with @/@ suffix) will permit access to individual collections. + +* The request @GET /arvados/v1/collections/962eh-4zz18-xi32mpz2621o8km@ will succeed +* Be aware that requests for listing @GET /arvados/v1/collections@ (no @/@ suffix) will be rejected, because it is not a match with the rule @GET /arvados/v1/collections/@ +* A listing request @GET /arvados/v1/collections/@ will have the trailing @/@ suffix trimmed before the scope check, as a result it will not match the rule @GET /arvados/v1/collections/@. + +To allow both listing objects and requesting individual objects, include both in the scope: @["GET /arvados/v1/collections", "GET /arvados/v1/collections/"]@ + +A narrow scope such as @GET /arvados/v1/collections/962eh-4zz18-xi32mpz2621o8km@ will disallow listing objects as well as disallow requesting any object other than those listed in the scope. diff --git a/doc/images/Arvados_Permissions.svg b/doc/images/Arvados_Permissions.svg new file mode 100644 index 0000000000..018a172c25 --- /dev/null +++ b/doc/images/Arvados_Permissions.svg @@ -0,0 +1,4 @@ + + + + diff --git a/doc/images/Crunch_dispatch.svg b/doc/images/Crunch_dispatch.svg new file mode 100644 index 0000000000..aa63962edb --- /dev/null +++ b/doc/images/Crunch_dispatch.svg @@ -0,0 +1,4 @@ + + + + diff --git a/doc/images/Keep_manifests.svg b/doc/images/Keep_manifests.svg new file mode 100644 index 0000000000..66a81620ba --- /dev/null +++ b/doc/images/Keep_manifests.svg @@ -0,0 +1,4 @@ + + + + diff --git a/doc/images/Keep_reading_writing_block.svg b/doc/images/Keep_reading_writing_block.svg new file mode 100644 index 0000000000..3333eabbc5 --- /dev/null +++ b/doc/images/Keep_reading_writing_block.svg @@ -0,0 +1,4 @@ + + + + diff --git a/doc/images/Keep_rendezvous_hashing.svg b/doc/images/Keep_rendezvous_hashing.svg new file mode 100644 index 0000000000..0c5c04ba8d --- /dev/null +++ b/doc/images/Keep_rendezvous_hashing.svg @@ -0,0 +1,4 @@ + + + + diff --git a/doc/images/Session_Establishment.svg b/doc/images/Session_Establishment.svg new file mode 100644 index 0000000000..e60cb9b6d4 --- /dev/null +++ b/doc/images/Session_Establishment.svg @@ -0,0 +1,4 @@ + + + + diff --git a/doc/install/crunch2-slurm/install-prerequisites.html.textile.liquid b/doc/install/crunch2-slurm/install-prerequisites.html.textile.liquid index c4dc92940b..26c7dde361 100644 --- a/doc/install/crunch2-slurm/install-prerequisites.html.textile.liquid +++ b/doc/install/crunch2-slurm/install-prerequisites.html.textile.liquid @@ -1,9 +1,9 @@ --- layout: default navsection: installguide -title: Crunch v2 SLURM prerequisites +title: Containers API SLURM prerequisites ... -Crunch v2 containers can be dispatched to a SLURM cluster. The dispatcher sends work to the cluster using SLURM's @sbatch@ command, so it works in a variety of SLURM configurations. +Containers can be dispatched to a SLURM cluster. The dispatcher sends work to the cluster using SLURM's @sbatch@ command, so it works in a variety of SLURM configurations. In order to run containers, you must run the dispatcher as a user that has permission to set up FUSE mounts and run Docker containers on each compute node. This install guide refers to this user as the @crunch@ user. We recommend you create this user on each compute node with the same UID and GID, and add it to the @fuse@ and @docker@ system groups to grant it the necessary permissions. However, you can run the dispatcher under any account with sufficient permissions across the cluster. diff --git a/doc/sdk/cli/install.html.textile.liquid b/doc/sdk/cli/install.html.textile.liquid index 8cde514f68..4776a97895 100644 --- a/doc/sdk/cli/install.html.textile.liquid +++ b/doc/sdk/cli/install.html.textile.liquid @@ -3,7 +3,6 @@ layout: default navsection: sdk navmenu: CLI title: "Installation" - ... Arvados CLI tools are written in Ruby and Python. To use the @arv@ command, you can either install the @arvados-cli@ gem via RubyGems or build and install the package from source. The @arv@ command also relies on other Arvados tools. To get those, install the @arvados-python-client@ and @arvados-cwl-runner@ packages, either from PyPI or source. diff --git a/doc/sdk/go/example.html.textile.liquid b/doc/sdk/go/example.html.textile.liquid new file mode 100644 index 0000000000..5fe202dbff --- /dev/null +++ b/doc/sdk/go/example.html.textile.liquid @@ -0,0 +1,76 @@ +--- +layout: default +navsection: sdk +navmenu: Python +title: Examples +... + +See "Arvados GoDoc":https://godoc.org/git.curoverse.com/arvados.git/sdk/go for detailed documentation. + +In these examples, the site prefix is @aaaaa@. + +h2. Initialize SDK + +
+import (
+  "git.curoverse.com/arvados.git/sdk/go/arvados"
+  "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+}
+
+func main() {
+  arv, err := arvadosclient.MakeArvadosClient()
+  if err != nil {
+    log.Fatalf("Error setting up arvados client %s", err.Error())
+  }
+}
+
+ +h2. create + +
+  var collection arvados.Collection
+  err := api.Create("collections", Dict{"collection": Dict{"name": "create example"}}, &collection)
+
+ +h2. delete + +
+  var collection arvados.Collection
+  err := api.Delete("collections", "aaaaa-4zz18-ccccccccccccccc", Dict{}, &collection)
+
+ +h2. get + +
+  var collection arvados.Collection
+  err := api.Get("collections", "aaaaa-4zz18-ccccccccccccccc", Dict{}, &collection)
+
+ +h2. list + +
+  var collection arvados.Collection
+  err := api.List("collections", Dict{}, &collection)
+
+ +h2. update + +
+  var collection arvados.Collection
+  err := api.Update("collections", "aaaaa-4zz18-ccccccccccccccc", Dict{"collection": Dict{"name": "update example"}}, &collection)
+
+ +h2. Get current user + +
+  var user arvados.User
+  err := api.Get("users", "current", Dict{}, &user)
+
+ +h2. Example program + +You can save this source as a .go file and run it: + +{% code 'example_sdk_go' as go %} + +A few more usage examples can be found in the "services/keepproxy":https://dev.arvados.org/projects/arvados/repository/revisions/master/show/services/keepproxy and "sdk/go/keepclient":https://dev.arvados.org/projects/arvados/repository/revisions/master/show/sdk/go/keepclient directories in the arvados source tree. diff --git a/doc/sdk/go/index.html.textile.liquid b/doc/sdk/go/index.html.textile.liquid index 24873318da..81f4f9914b 100644 --- a/doc/sdk/go/index.html.textile.liquid +++ b/doc/sdk/go/index.html.textile.liquid @@ -2,24 +2,17 @@ layout: default navsection: sdk navmenu: Go -title: "Go SDK" - +title: "Installation" ... The Go ("Golang":http://golang.org) SDK provides a generic set of wrappers so you can make API calls easily. +See "Arvados GoDoc":https://godoc.org/git.curoverse.com/arvados.git/sdk/go for detailed documentation. + h3. Installation -You don't need to install anything. Just import the client like this. The go tools will fetch the relevant code and dependencies for you. +Use @go get git.curoverse.com/arvados.git/sdk/go/arvadosclient@. The go tools will fetch the relevant code and dependencies for you. {% code 'example_sdk_go_imports' as go %} If you need pre-release client code, you can use the latest version from the repo by following "these instructions.":https://dev.arvados.org/projects/arvados/wiki/Go#Using-Go-with-Arvados - -h3. Example - -You can save this source as a .go file and run it: - -{% code 'example_sdk_go' as go %} - -A few more usage examples can be found in the "services/keepproxy":https://dev.arvados.org/projects/arvados/repository/revisions/master/show/services/keepproxy and "sdk/go/keepclient":https://dev.arvados.org/projects/arvados/repository/revisions/master/show/sdk/go/keepclient directories in the arvados source tree. diff --git a/doc/sdk/index.html.textile.liquid b/doc/sdk/index.html.textile.liquid index db5d6f13b0..7633228995 100644 --- a/doc/sdk/index.html.textile.liquid +++ b/doc/sdk/index.html.textile.liquid @@ -1,19 +1,16 @@ --- layout: default navsection: sdk -title: "Arvados SDK Reference" +title: "SDK Reference" ... -This section documents how to access the Arvados API and Keep using various programming languages. +This section documents language bindings for the "Arvados API":{{site.baseurl}}/api and Keep that are available for various programming languages. Not all features are available in every SDK. The most complete SDK is the Python SDK. Note that this section only gives a high level overview of each SDK. Consult the "Arvados API":{{site.baseurl}}/api section for detailed documentation about Arvados API calls available on each resource. * "Python SDK":{{site.baseurl}}/sdk/python/sdk-python.html +* "Command line SDK":{{site.baseurl}}/sdk/cli/install.html ("arv") +* "Go SDK":{{site.baseurl}}/sdk/go/index.html * "Perl SDK":{{site.baseurl}}/sdk/perl/index.html * "Ruby SDK":{{site.baseurl}}/sdk/ruby/index.html * "Java SDK":{{site.baseurl}}/sdk/java/index.html -* "Go SDK":{{site.baseurl}}/sdk/go/index.html -* "Command line SDK":{{site.baseurl}}/sdk/cli/index.html ("arv") - -SDKs not yet implemented: -* Rails SDK: Workbench uses an ActiveRecord-like interface to Arvados. This hasn't yet been extracted from Workbench and packaged as a gem. -* R: We plan to support this, but it has not been implemented yet. +Many Arvados Workbench pages, under the the *Advanced* tab, provide examples of API and SDK use for accessing the current resource . diff --git a/doc/sdk/java/example.html.textile.liquid b/doc/sdk/java/example.html.textile.liquid new file mode 100644 index 0000000000..9ff848b94e --- /dev/null +++ b/doc/sdk/java/example.html.textile.liquid @@ -0,0 +1,78 @@ +--- +layout: default +navsection: sdk +navmenu: Java +title: "Examples" +... + +h2. Initialize SDK + +
+import org.arvados.sdk.Arvados;
+
+ +
+    String apiName = "arvados";
+    String apiVersion = "v1";
+
+    Arvados arv = new Arvados(apiName, apiVersion);
+
+ +h2. create + +
+    Map collection = new HashMap();
+    collection.put("name", "create example");
+
+    Map params = new HashMap();
+    params.put("collection", collection);
+    Map response = arv.call("collections", "create", params);
+
+ +h2. delete + +
+    Map params = new HashMap();
+    params.put("uuid", uuid);
+    Map response = arv.call("collections", "delete", params);
+
+ +h2. get + +
+    params = new HashMap();
+    params.put("uuid", userUuid);
+    Map response = arv.call("users", "get", params);
+
+ +h2. list + +
+    Map params = new HashMap();
+    Map response = arv.call("users", "list", params);
+
+    // get uuid of the first user from the response
+    List items = (List)response.get("items");
+
+    Map firstUser = (Map)items.get(0);
+    String userUuid = (String)firstUser.get("uuid");
+
+ +h2. update + +
+    Map collection = new HashMap();
+    collection.put("name", "update example");
+
+    Map params = new HashMap();
+    params.put("uuid", uuid);
+    params.put("collection", collection);
+    Map response = arv.call("collections", "update", params);
+
+ +h2. Get current user + +
+    Map params = new HashMap();
+    Map response = arv.call("users", "current", params);
+
diff --git a/doc/sdk/java/index.html.textile.liquid b/doc/sdk/java/index.html.textile.liquid index 48f72d3a7d..27984b2fe4 100644 --- a/doc/sdk/java/index.html.textile.liquid +++ b/doc/sdk/java/index.html.textile.liquid @@ -2,8 +2,7 @@ layout: default navsection: sdk navmenu: Java -title: "Java SDK" - +title: "Installation" ... The Java SDK provides a generic set of wrappers so you can make API calls in java. @@ -11,10 +10,10 @@ The Java SDK provides a generic set of wrappers so you can make API calls in jav h3. Introdution * The Java SDK requires Java 6 or later - + * The Java SDK is implemented as a maven project. Hence, you would need a working maven environment to be able to build the source code. If you do not have maven setup, -you may find the "Maven in 5 Minutes":http://maven.apache.org/guides/getting-started/maven-in-five-minutes.html link useful. +you may find the "Maven in 5 Minutes":http://maven.apache.org/guides/getting-started/maven-in-five-minutes.html link useful. * In this document $ARVADOS_HOME is used to refer to the directory where arvados code is cloned in your system. For ex: $ARVADOS_HOME = $HOME/arvados @@ -38,7 +37,7 @@ ARVADOS_API_HOST_INSECURE: Set this to true if you are using self-signed * Please see "api-tokens":{{site.baseurl}}/user/reference/api-tokens.html for full details. - + h3. Building the Arvados SDK @@ -124,7 +123,7 @@ The local repository is usually located in your home directory at diff --git a/doc/sdk/perl/example.html.textile.liquid b/doc/sdk/perl/example.html.textile.liquid new file mode 100644 index 0000000000..980129cf13 --- /dev/null +++ b/doc/sdk/perl/example.html.textile.liquid @@ -0,0 +1,81 @@ +--- +layout: default +navsection: sdk +navmenu: Perl +title: "Examples" +... + +h2. Initialize SDK + +Set up an API client user agent: + + +

+use Arvados;
+my $arv = Arvados->new('apiVersion' => 'v1');
+
+ + +The SDK retrieves the list of API methods from the server at run time. Therefore, the set of available methods is determined by the server version rather than the SDK version. + +h2. create + +Create an object: + + +
my $test_link = $arv->{'links'}->{'create'}->execute('link' => { 'link_class' => 'test', 'name' => 'test' });
+
+ + +h2. delete + + +
my $some_user = $arv->{'collections'}->{'get'}->execute('uuid' => $collection_uuid);
+
+ + +h2. get + +Retrieve an object by ID: + + +
my $some_user = $arv->{'users'}->{'get'}->execute('uuid' => $current_user_uuid);
+
+ + +Get the UUID of an object that was retrieved using the SDK: + + +
my $current_user_uuid = $current_user->{'uuid'}
+
+ + +h2. list + +Get a list of objects: + + +
my $repos = $arv->{'repositories'}->{'list'}->execute;
+print ("UUID of first repo returned is ", $repos->{'items'}->[0], "\n");
+
+ + +h2. update + +Update an object: + + +
my $test_link = $arv->{'links'}->{'update'}->execute(
+        'uuid' => $test_link->{'uuid'},
+        'link' => { 'properties' => { 'foo' => 'bar' } });
+
+ + +h2. Get current user + +Get the User object for the current user: + + +
my $current_user = $arv->{'users'}->{'current'}->execute;
+
+ diff --git a/doc/sdk/perl/index.html.textile.liquid b/doc/sdk/perl/index.html.textile.liquid index e28d02011b..407008d1f7 100644 --- a/doc/sdk/perl/index.html.textile.liquid +++ b/doc/sdk/perl/index.html.textile.liquid @@ -2,8 +2,7 @@ layout: default navsection: sdk navmenu: Perl -title: "Perl SDK" - +title: "Installation" ... The Perl SDK provides a generic set of wrappers so you can make API calls easily. @@ -63,59 +62,3 @@ EOF arvados.v1.users.current.full_name = 'Your Name' - -h3. Examples - -Set up an API client user agent: - - -
my $arv = Arvados->new('apiVersion' => 'v1');
-
- - -Get the User object for the current user: - - -
my $current_user = $arv->{'users'}->{'current'}->execute;
-
- - -Get the UUID of an object that was retrieved using the SDK: - - -
my $current_user_uuid = $current_user->{'uuid'}
-
- - -Retrieve an object by ID: - - -
my $some_user = $arv->{'users'}->{'get'}->execute('uuid' => $current_user_uuid);
-
- - -Create an object: - - -
my $test_link = $arv->{'links'}->{'create'}->execute('link' => { 'link_class' => 'test', 'name' => 'test' });
-
- - -Update an object: - - -
my $test_link = $arv->{'links'}->{'update'}->execute(
-        'uuid' => $test_link->{'uuid'},
-        'link' => { 'properties' => { 'foo' => 'bar' } });
-
- - -Get a list of objects: - - -
my $repos = $arv->{'repositories'}->{'list'}->execute;
-print ("UUID of first repo returned is ", $repos->{'items'}->[0], "\n");
-
- - -The SDK retrieves the list of API methods from the server at run time. Therefore, the set of available methods is determined by the server version rather than the SDK version. diff --git a/doc/sdk/python/events.html.textile.liquid b/doc/sdk/python/events.html.textile.liquid index 6afb9b51a3..0988a7ce9c 100644 --- a/doc/sdk/python/events.html.textile.liquid +++ b/doc/sdk/python/events.html.textile.liquid @@ -5,7 +5,7 @@ navmenu: Python title: Subscribing to events ... -Arvados applications can subscribe to a live event stream from the database. Events are described in the "Log record schema.":{{site.baseurl}}/api/schema/Log.html +Arvados applications can subscribe to a live event stream from the database. Events are described in the "Log resource.":{{site.baseurl}}/api/methods/logs.html {% code 'events_py' as python %} diff --git a/doc/sdk/python/example.html.textile.liquid b/doc/sdk/python/example.html.textile.liquid new file mode 100644 index 0000000000..e91055e101 --- /dev/null +++ b/doc/sdk/python/example.html.textile.liquid @@ -0,0 +1,51 @@ +--- +layout: default +navsection: sdk +navmenu: Python +title: Examples +... + +In these examples, the site prefix is @aaaaa@. + +h2. Initialize SDK + +
+import arvados
+api = arvados.api("v1")
+
+ +h2. create + +
+result = api.collection().create(body={"collection": {"name": "create example"}}).execute()
+
+ +h2. delete + +
+result = api.collections().delete(uuid="aaaaa-4zz18-ccccccccccccccc").execute()
+
+ +h2. get + +
+result = api.collections().get(uuid="aaaaa-4zz18-ccccccccccccccc").execute()
+
+ +h2. list + +
+result = api.collections().list(filters=[["uuid", "=", "aaaaa-bbbbb-ccccccccccccccc"]]).execute()
+
+ +h2. update + +
+result = api.collections().update(uuid="aaaaa-4zz18-ccccccccccccccc", body={"collection": {"name": "update example"}}).execute()
+
+ +h2. Get current user + +
+result = api.users().current().execute()
+
diff --git a/doc/sdk/python/sdk-python.html.textile.liquid b/doc/sdk/python/sdk-python.html.textile.liquid index 0b0f77d377..e9b560a965 100644 --- a/doc/sdk/python/sdk-python.html.textile.liquid +++ b/doc/sdk/python/sdk-python.html.textile.liquid @@ -2,8 +2,7 @@ layout: default navsection: sdk navmenu: Python -title: "Python SDK" - +title: "Installation" ... The Python SDK provides access from Python to the Arvados API and Keep. It also includes a number of command line tools for using and administering Arvados and Keep, and some conveniences for use in Crunch scripts; see "Crunch utility libraries":crunch-utility-libraries.html for details. diff --git a/doc/sdk/ruby/example.html.textile.liquid b/doc/sdk/ruby/example.html.textile.liquid new file mode 100644 index 0000000000..409a70a813 --- /dev/null +++ b/doc/sdk/ruby/example.html.textile.liquid @@ -0,0 +1,75 @@ +--- +layout: default +navsection: sdk +navmenu: Python +title: Examples +... + +h2. Initialize SDK + +Import the module and set up an API client user agent: + +
+require 'arvados'
+arv = Arvados.new(apiVersion: 'v1')
+
+ +The SDK retrieves the list of API methods from the server at run time. Therefore, the set of available methods is determined by the server version rather than the SDK version. + +h2. create + +Create an object: + +
+new_link = arv.link.create(link: {link_class: 'test', name: 'test'})
+
+ +h2. delete + +Delete an object: + +
+arv.link.delete(uuid: new_link[:uuid])
+
+ +h2. get + +Retrieve an object by ID: + +
+some_user = arv.user.get(uuid: current_user_uuid)
+
+ +h2. list + +Get a list of objects: + +
+repos = arv.repository.list
+first_repo = repos[:items][0]
+puts "UUID of first repo returned is #{first_repo[:uuid]}"
+UUID of first repo returned is qr1hi-s0uqq-b1bnybpx3u5temz
+
+ +h2. update + +Update an object: + +
+updated_link = arv.link.update(uuid: new_link[:uuid],
+                               link: {properties: {foo: 'bar'}})
+
+ +h2. Get current user + +Get the User object for the current user: + +
+current_user = arv.user.current
+
+ +Get the UUID of an object that was retrieved using the SDK: + +
+current_user_uuid = current_user[:uuid]
+
diff --git a/doc/sdk/ruby/index.html.textile.liquid b/doc/sdk/ruby/index.html.textile.liquid index b78a37d03a..5ad02543ef 100644 --- a/doc/sdk/ruby/index.html.textile.liquid +++ b/doc/sdk/ruby/index.html.textile.liquid @@ -2,8 +2,7 @@ layout: default navsection: sdk navmenu: Ruby -title: "Ruby SDK" - +title: "Installation" ... The Ruby SDK provides a generic set of wrappers so you can make API calls easily. @@ -52,74 +51,3 @@ EOF arvados.v1.users.current.full_name = 'Your Name' - -h3. Examples - -Import the module (we skipped this step above by using "ruby -r arvados"): - - -
require 'arvados'
-
- - -Set up an API client user agent: - - -
arv = Arvados.new(apiVersion: 'v1')
-
- - -Get the User object for the current user: - - -
current_user = arv.user.current
-
- - -Get the UUID of an object that was retrieved using the SDK: - - -
current_user_uuid = current_user[:uuid]
-
- - -Retrieve an object by ID: - - -
some_user = arv.user.get(uuid: current_user_uuid)
-
- - -Create an object: - - -
new_link = arv.link.create(link: {link_class: 'test', name: 'test'})
-
- - -Update an object: - - -
updated_link = arv.link.update(uuid: new_link[:uuid],
-                               link: {properties: {foo: 'bar'}})
-
- - -Delete an object: - - -
arv.link.delete(uuid: new_link[:uuid])
-
- - -Get a list of objects: - - -
repos = arv.repository.list
-first_repo = repos[:items][0]
-puts "UUID of first repo returned is #{first_repo[:uuid]}"
-UUID of first repo returned is qr1hi-s0uqq-b1bnybpx3u5temz
-
- - -The SDK retrieves the list of API methods from the server at run time. Therefore, the set of available methods is determined by the server version rather than the SDK version. diff --git a/doc/user/cwl/cwl-runner.html.textile.liquid b/doc/user/cwl/cwl-runner.html.textile.liquid index c00f475ad9..6dac43aa65 100644 --- a/doc/user/cwl/cwl-runner.html.textile.liquid +++ b/doc/user/cwl/cwl-runner.html.textile.liquid @@ -15,6 +15,7 @@ The @arvados-cwl-runner@ client is installed by default on Arvados shell nodes. 
~$ virtualenv ~/venv
 ~$ . ~/venv/bin/activate
+~$ pip install -U setuptools
 ~$ pip install arvados-cwl-runner
diff --git a/doc/user/reference/job-pipeline-ref.html.textile.liquid b/doc/user/reference/job-pipeline-ref.html.textile.liquid index f8f749c7cb..643e6bb3df 100644 --- a/doc/user/reference/job-pipeline-ref.html.textile.liquid +++ b/doc/user/reference/job-pipeline-ref.html.textile.liquid @@ -4,4 +4,4 @@ navsection: userguide title: "Pipeline template reference" ... -Pipeline template options are described on the "pipeline template schema page.":{{site.baseurl}}/api/schema/PipelineTemplate.html +Pipeline template options are described on the "pipeline template schema page.":{{site.baseurl}}/api/methods/pipeline_templates.html diff --git a/doc/user/tutorials/running-external-program.html.textile.liquid b/doc/user/tutorials/running-external-program.html.textile.liquid index ef4634ee74..a7682594c9 100644 --- a/doc/user/tutorials/running-external-program.html.textile.liquid +++ b/doc/user/tutorials/running-external-program.html.textile.liquid @@ -22,7 +22,7 @@ This will open the template record in an interactive text editor (as specified b * @"name"@ is a human-readable name for the pipeline. * @"components"@ is a set of scripts or commands that make up the pipeline. Each component is given an identifier (@"bwa-mem"@ and @"SortSam"@) in this example). -** Each entry in components @"components"@ is an Arvados job submission. For more information about individual jobs, see the "job object reference":{{site.baseurl}}/api/schema/Job.html and "job create method.":{{site.baseurl}}/api/methods/jobs.html#create +** Each entry in components @"components"@ is an Arvados job submission. For more information about individual jobs, see the "job resource reference.":{{site.baseurl}}/api/methods/jobs.html * @"repository"@, @"script_version"@, and @"script"@ indicate that we intend to use the external @"run-command"@ tool wrapper that is part of the Arvados. These parameters are described in more detail in "Writing a script":tutorial-firstscript.html. * @"runtime_constraints"@ describes runtime resource requirements for the component. ** @"docker_image"@ specifies the "Docker":https://www.docker.com/ runtime environment in which to run the job. The Docker image @"bcosc/arv-base-java"@ supplied here has the Java runtime environment, bwa, and samtools installed. @@ -62,7 +62,7 @@ Test data is available in the "Arvados Tutorial":{{site.arvados_workbench_host}} * Choose "Tutorial chromosome 19 reference (2463fa9efeb75e099685528b3b9071e0+438)":{{site.arvados_workbench_host}}/collections/2463fa9efeb75e099685528b3b9071e0+438 for the "reference_collection" parameter * Choose "Tutorial sample exome (3229739b505d2b878b62aed09895a55a+142)":{{site.arvados_workbench_host}}/collections/3229739b505d2b878b62aed09895a55a+142 for the "sample" parameter -For more information and examples for writing pipelines, see the "pipeline template reference":{{site.baseurl}}/api/schema/PipelineTemplate.html +For more information and examples for writing pipelines, see the "pipeline template reference":{{site.baseurl}}/api/methods/pipeline_templates.html h2. Re-using your pipeline run diff --git a/doc/user/tutorials/tutorial-submit-job.html.textile.liquid b/doc/user/tutorials/tutorial-submit-job.html.textile.liquid index 47e8dc750c..d1556cfe02 100644 --- a/doc/user/tutorials/tutorial-submit-job.html.textile.liquid +++ b/doc/user/tutorials/tutorial-submit-job.html.textile.liquid @@ -81,10 +81,10 @@ In the editor, enter the following template: * @"repository"@ is the name of a git repository to search for the script version. You can access a list of available git repositories on the Arvados Workbench in the *Repositories* page using the top navigation menu icon. * @"script_version"@ specifies the version of the script that you wish to run. This can be in the form of an explicit Git revision hash, a tag, or a branch (in which case it will use the HEAD of the specified branch). Arvados logs the script version that was used in the run, enabling you to go back and re-run any past job with the guarantee that the exact same code will be used as was used in the previous run. * @"script"@ specifies the filename of the script to run. Crunch expects to find this in the @crunch_scripts/@ subdirectory of the Git repository. -* @"runtime_constraints"@ describes the runtime environment required to run the job. These are described in the "job record schema":{{site.baseurl}}/api/schema/Job.html +* @"runtime_constraints"@ describes the runtime environment required to run the job. These are described in the "job record schema":{{site.baseurl}}/api/methods/jobs.html h2. Running your pipeline Your new pipeline template should appear at the top of the Workbench "pipeline templates":{{site.arvados_workbench_host}}/pipeline_templates page. You can run your pipeline "using Workbench":tutorial-pipeline-workbench.html or the "command line.":{{site.baseurl}}/user/topics/running-pipeline-command-line.html -For more information and examples for writing pipelines, see the "pipeline template reference":{{site.baseurl}}/api/schema/PipelineTemplate.html +For more information and examples for writing pipelines, see the "pipeline template reference":{{site.baseurl}}/api/methods/pipeline_templates.html